diff --git a/roles/float-infra-nginx/templates/config/conf.d/noindex.conf b/roles/float-infra-nginx/templates/config/conf.d/noindex.conf new file mode 100644 index 0000000000000000000000000000000000000000..b0b89d1a259849d2bf97042bdac3c6b1035b2f2b --- /dev/null +++ b/roles/float-infra-nginx/templates/config/conf.d/noindex.conf @@ -0,0 +1,9 @@ +# Global map used to forcefully prevent indexing of specific URLs. +# Configure via drop-in files in /etc/nginx/noindex/. + +map $host$uri $noindex_header { + default ''; + + include /etc/nginx/noindex/*.conf; +} + diff --git a/roles/float-infra-nginx/templates/config/noindex/README.md b/roles/float-infra-nginx/templates/config/noindex/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dcaf7b88509d89885d5d5f77e4dbdb0e0dd00df8 --- /dev/null +++ b/roles/float-infra-nginx/templates/config/noindex/README.md @@ -0,0 +1,32 @@ +URL deindexing +=== + +This list allows you to prevent bots from indexing specific URLs. +The string that is matched is the combination of the hostname and +the request path (without query arguments), i.e. for a request to + +``` +https://example.com/path/?arg=value +``` + +this map will attempt to match + +``` +example.com/path/ +``` + +To deindex a URL, create or modify a file with a *.conf* extension in +this directory. The files should contain one entry per line, with the +[NGINX map module +syntax](http://nginx.org/en/docs/http/ngx_http_map_module.html): + +``` +"example.com/path/" noindex; +``` + +The value should always be set to `noindex`. + +Deindexing via HTTP header is described here: + +https://developers.google.com/search/docs/advanced/robots/robots_meta_tag + diff --git a/roles/float-infra-nginx/templates/config/snippets/site-common.conf b/roles/float-infra-nginx/templates/config/snippets/site-common.conf index 72edd4864643d47d58d63923aba1c942cbcc5bc1..905d18bd840123c4441850733c8c3e02a203f0f7 100644 --- a/roles/float-infra-nginx/templates/config/snippets/site-common.conf +++ b/roles/float-infra-nginx/templates/config/snippets/site-common.conf @@ -75,3 +75,6 @@ location {{ ep.path }}/ { } {% endfor %} {% endfor %} + +add_header X-Robots-Tag $noindex_header; +