Files
ipranges/googlebot/downloader.sh
Gavin D. Howard 4d8d13982c Separate GoogleBot from Google (#7)
This is for the purpose of allowing it while prohibiting the other
Google IP addresses.

Signed-off-by: Gavin D. Howard <gavin@gavinhoward.com>
2024-01-16 10:13:25 +03:00

23 lines
627 B
Bash
Executable File

#!/bin/bash
set -euo pipefail
set -x
# Public GoogleBot IP ranges
# From: https://developers.google.com/search/docs/advanced/crawling/verifying-googlebot
curl -s https://developers.google.com/search/apis/ipranges/googlebot.json > /tmp/googlebot.json
# save ipv4
jq '.prefixes[] | [.ipv4Prefix][] | select(. != null)' -r /tmp/googlebot.json >> /tmp/googlebot-ipv4.txt
# save ipv6
jq '.prefixes[] | [.ipv6Prefix][] | select(. != null)' -r /tmp/googlebot.json >> /tmp/googlebot-ipv6.txt
# sort & uniq
sort -V /tmp/googlebot-ipv4.txt | uniq > googlebot/ipv4.txt
sort -V /tmp/googlebot-ipv6.txt | uniq > googlebot/ipv6.txt