Browse Source

Exclude Cloudflare instances from services.json by default

This updates the services json file to exclude all instances that are
detected to be using Cloudflare nameservers.

A separate "services-full.json" file will continue to be tracked in the
repo, which will include the full list of all instances for each
service and can be used with the `FARSIDE_SERVICES_JSON` environment
variable for anyone wanting to access the full instance list for each
service.

See #43
Ben Busby 2 năm trước cách đây
mục cha
commit
00225fdbc6
3 tập tin đã thay đổi với 391 bổ sung80 xóa
  1. 32 1
      .github/workflows/update-instances.yml
  2. 356 0
      services-full.json
  3. 3 79
      services.json

+ 32 - 1
.github/workflows/update-instances.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Install dependencies
-      run: sudo apt-get install -y jq
+      run: sudo apt-get install -y jq dnsutils
 
     - uses: webfactory/ssh-agent@v0.5.3
       with:
@@ -27,6 +27,32 @@ jobs:
           sed -i 's/\/"/"/g' services.json
         }
 
+        function remove_cf_instances() {
+          rm out.json
+          file="services.json"
+
+          while read -r line; do
+              if [[ "$line" == "\"https://"* ]]; then
+                  domain=$(echo "$line" | sed -e "s/^\"https:\/\///" -e "s/\",//" -e "s/\"//")
+                  ns=$(dig ns "$domain")
+                  if [[ "$ns" == *"cloudflare"* ]]; then
+                      echo "\"$domain\" using cloudflare, skipping..."
+                  else
+                      echo "$line" >> out.json
+                  fi
+              else
+                  echo "$line" >> out.json
+              fi
+          done <$file
+
+          # Remove any trailing commas from new instance lists
+          sed -i '' -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json
+
+          mv services.json services-full.json
+          cat out.json | jq --indent 2 . > services.json
+          rm out.json
+        }
+
         # ==============================================================
         # Git config
         # ==============================================================
@@ -173,6 +199,11 @@ jobs:
         # TODO: Update instances for other services
         # ==============================================================
 
+        # ==============================================================
+        # Remove cloudflare instances
+        # ==============================================================
+        remove_cf_instances
+
         # ==============================================================
         # Push changes
         # ==============================================================

+ 356 - 0
services-full.json

@@ -0,0 +1,356 @@
+[
+  {
+    "type": "libreddit",
+    "test_url": "/r/popular",
+    "fallback": "https://libredd.it",
+    "instances": [
+      "https://libreddit.albony.xyz",
+      "https://libredd.it",
+      "https://libreddit.spike.codes",
+      "https://libreddit.dothq.co",
+      "https://libreddit.kavin.rocks",
+      "https://libreddit.40two.app",
+      "https://reddit.invak.id",
+      "https://reddit.phii.me",
+      "https://lr.riverside.rocks",
+      "https://libreddit.silkky.cloud",
+      "https://libreddit.database.red",
+      "https://libreddit.exonip.de",
+      "https://libreddit.domain.glass",
+      "https://libreddit.sugoma.tk",
+      "https://libreddit.trevorthalacker.com",
+      "https://reddit.artemislena.eu",
+      "https://r.nf",
+      "https://libreddit.some-things.org",
+      "https://reddit.stuehieyr.com",
+      "https://lr.mint.lgbt",
+      "https://libreddit.igna.rocks",
+      "https://libreddit.autarkic.org",
+      "https://libreddit.flux.industries",
+      "https://libreddit.drivet.xyz",
+      "https://libreddit.de",
+      "https://libreddit.bus-hit.me",
+      "https://leddit.xyz",
+      "https://de.leddit.xyz"
+    ]
+  },
+  {
+    "type": "teddit",
+    "test_url": "/r/popular",
+    "fallback": "https://teddit.net",
+    "instances": [
+      "https://i.opnxng.com",
+      "https://incogsnoo.com",
+      "https://rdt.trom.tf",
+      "https://reddit.lol",
+      "https://snoo.ioens.is",
+      "https://tedd.it",
+      "https://teddit.adminforge.de",
+      "https://teddit.bus-hit.me",
+      "https://teddit.domain.glass",
+      "https://teddit.encrypted-data.xyz",
+      "https://teddit.froth.zone",
+      "https://teddit.garudalinux.org",
+      "https://teddit.ggc-project.de",
+      "https://teddit.httpjames.space",
+      "https://teddit.namazso.eu",
+      "https://teddit.nautolan.racing",
+      "https://teddit.net",
+      "https://teddit.pussthecat.org",
+      "https://teddit.sethforprivacy.com",
+      "https://teddit.tinfoil-hat.net",
+      "https://teddit.tokhmi.xyz",
+      "https://teddit.totaldarkness.net",
+      "https://teddit.zaggy.nl"
+    ]
+  },
+  {
+    "type": "bibliogram",
+    "test_url": "/u/officialrickastley",
+    "fallback": "https://bibliogram.art",
+    "instances": [
+      "https://bibliogram.art",
+      "https://bibliogram.snopyta.org",
+      "https://bibliogram.pussthecat.org",
+      "https://bibliogram.1d4.us",
+      "https://insta.trom.tf",
+      "https://bibliogram.hamster.dance",
+      "https://bib.actionsack.com"
+    ]
+  },
+  {
+    "type": "invidious",
+    "test_url": "/watch?v=eBGIQ7ZuuiU",
+    "fallback": "https://invidious.snopyta.org",
+    "instances": [
+      "https://inv.bp.projectsegfau.lt",
+      "https://inv.privacy.com.de",
+      "https://inv.riverside.rocks",
+      "https://inv.vern.cc",
+      "https://invidious.esmailelbob.xyz",
+      "https://invidious.flokinet.to",
+      "https://invidious.kavin.rocks",
+      "https://invidious.namazso.eu",
+      "https://invidious.nerdvpn.de",
+      "https://invidious.osi.kr",
+      "https://invidious.projectsegfau.lt",
+      "https://invidious.sethforprivacy.com",
+      "https://invidious.slipfox.xyz",
+      "https://invidious.snopyta.org",
+      "https://invidious.tiekoetter.com",
+      "https://invidious.weblibre.org",
+      "https://vid.puffyan.us",
+      "https://y.com.sb",
+      "https://yewtu.be",
+      "https://youtube.076.ne.jp",
+      "https://yt.artemislena.eu"
+    ]
+  },
+  {
+    "type": "piped",
+    "test_url": "/watch?v=eBGIQ7ZuuiU",
+    "fallback": "https://piped.kavin.rocks",
+    "instances": [
+      "https://piped.kavin.rocks",
+      "https://piped.silkky.cloud",
+      "https://piped.tokhmi.xyz",
+      "https://piped.moomoo.me",
+      "https://piped.mint.lgbt"
+    ]
+  },
+  {
+    "type": "nitter",
+    "test_url": "/twitter",
+    "fallback": "https://nitter.net",
+    "instances": [
+      "https://nitter.net",
+      "https://nitter.42l.fr",
+      "https://nitter.pussthecat.org",
+      "https://nitter.nixnet.services",
+      "https://nitter.fdn.fr",
+      "https://nitter.1d4.us",
+      "https://nitter.kavin.rocks",
+      "https://nitter.vxempire.xyz",
+      "https://nitter.unixfox.eu",
+      "https://nitter.domain.glass",
+      "https://nitter.eu",
+      "https://nitter.namazso.eu",
+      "https://nitter.mailstation.de",
+      "https://nitter.actionsack.com",
+      "https://nitter.cattube.org",
+      "https://birdsite.xanny.family",
+      "https://nitter.hu",
+      "https://nitter.exonip.de",
+      "https://twitr.gq",
+      "https://nitter.moomoo.me",
+      "https://bird.trom.tf",
+      "https://nitter.it",
+      "https://twitter.censors.us",
+      "https://nitter.grimneko.de",
+      "https://nitter.koyu.space",
+      "https://nitter.ir",
+      "https://nitter.autarkic.org",
+      "https://n.0x0.st",
+      "https://n.hyperborea.cloud",
+      "https://nitter.ca",
+      "https://twitter.076.ne.jp",
+      "https://nitter.sethforprivacy.com",
+      "https://nitter.bus-hit.me",
+      "https://nttr.stream",
+      "https://de.nttr.stream",
+      "https://n.l5.ca"
+    ]
+  },
+  {
+    "type": "scribe",
+    "test_url": "/@ftrain/big-data-small-effort-b62607a43a8c",
+    "fallback": "https://scribe.rip",
+    "instances": [
+      "https://scribe.rip",
+      "https://scribe.nixnet.services",
+      "https://scribe.citizen4.eu",
+      "https://scribe.bus-hit.me",
+      "https://scribe.froth.zone"
+    ]
+  },
+  {
+    "type": "simplytranslate",
+    "test_url": "/?engine=google&text=simplicado",
+    "fallback": "https://simplytranslate.org",
+    "instances": [
+      "https://simplytranslate.esmailelbob.xyz",
+      "https://simplytranslate.org",
+      "https://simplytranslate.pussthecat.org",
+      "https://st.manerakai.com",
+      "https://st.privacydev.net",
+      "https://st.tokhmi.xyz",
+      "https://tl.vern.cc",
+      "https://translate.beparanoid.de",
+      "https://translate.bus-hit.me",
+      "https://translate.josias.dev",
+      "https://translate.namazso.eu",
+      "https://translate.northboot.xyz",
+      "https://translate.priv.pw",
+      "https://translate.riverside.rocks",
+      "https://translate.slipfox.xyz",
+      "https://translate.tiekoetter.com"
+    ]
+  },
+  {
+    "type": "lingva",
+    "test_url": "/auto/en/hola",
+    "fallback": "https://lingva.ml",
+    "instances": [
+      "https://lingva.ml",
+      "https://translate.igna.rocks",
+      "https://lingva.pussthecat.org"
+    ]
+  },
+  {
+    "type": "rimgo",
+    "test_url": "/a/H8M4rcp",
+    "fallback": "https://i.bcow.xyz",
+    "instances": [
+      "https://i.bcow.xyz",
+      "https://rimgo.pussthecat.org",
+      "https://img.riverside.rocks",
+      "https://rimgo.totaldarkness.net",
+      "https://rimgo.bus-hit.me"
+    ]
+  },
+  {
+    "type": "whoogle",
+    "test_url": "/search?cookies_disabled=1&q=<%=query%>",
+    "fallback": "https://whoogle.fossho.st",
+    "instances": [
+      "https://gowogle.voring.me",
+      "https://s.tokhmi.xyz",
+      "https://search.albony.xyz",
+      "https://search.dr460nf1r3.org",
+      "https://search.garudalinux.org",
+      "https://search.sethforprivacy.com",
+      "https://search.wef.lol",
+      "https://whoogle.dcs0.hu",
+      "https://whoogle.esmailelbob.xyz",
+      "https://whoogle.fossho.st",
+      "https://whoogle.lunar.icu",
+      "https://whoogle.privacydev.net",
+      "https://www.whooglesearch.ml"
+    ]
+  },
+  {
+    "type": "searx",
+    "test_url": "/search?q=<%=query%>",
+    "fallback": "https://searx.be",
+    "instances": [
+      "https://dynabyte.ca",
+      "https://search.neet.works",
+      "https://searx.divided-by-zero.eu",
+      "https://searx.gnu.style",
+      "https://searx.rasp.fr",
+      "https://searx.ru",
+      "https://searx.sp-codes.de",
+      "https://searx.stuehieyr.com",
+      "https://searx.tuxcloud.net",
+      "https://searx.tyil.nl",
+      "https://searx.webheberg.info",
+      "https://searx.xyz",
+      "https://www.webrats.xyz"
+    ]
+  },
+  {
+    "type": "searxng",
+    "test_url": "/search?q=<%=query%>",
+    "fallback": "https://searx.be",
+    "instances": [
+      "https://darmarit.org/searx",
+      "https://de.xcxc.ml",
+      "https://etsi.me",
+      "https://jackgoss.xyz",
+      "https://northboot.xyz",
+      "https://opnxng.com",
+      "https://paulgo.io",
+      "https://priv.au",
+      "https://s.frlt.one",
+      "https://s.zhaocloud.net",
+      "https://saber.tk",
+      "https://search.ashs.club",
+      "https://search.bus-hit.me",
+      "https://search.chemicals-in-the-water.eu",
+      "https://search.gcomm.ch",
+      "https://search.mdosch.de",
+      "https://search.neet.works",
+      "https://search.ononoki.org",
+      "https://search.privacyguides.net",
+      "https://search.projectsegfau.lt",
+      "https://search.rabbit-company.com",
+      "https://search.sapti.me",
+      "https://search.teamriverbubbles.com",
+      "https://search.unlocked.link",
+      "https://search.zzls.xyz",
+      "https://searx.be",
+      "https://searx.becomesovran.com",
+      "https://searx.chocoflan.net",
+      "https://searx.ebnar.xyz",
+      "https://searx.ericaftereric.top",
+      "https://searx.fmac.xyz",
+      "https://searx.mha.fi",
+      "https://searx.mistli.net",
+      "https://searx.namejeff.xyz",
+      "https://searx.priv.pw",
+      "https://searx.prvcy.eu",
+      "https://searx.sev.monster",
+      "https://searx.slipfox.xyz/searx",
+      "https://searx.tiekoetter.com",
+      "https://searx.vimproved.me",
+      "https://searx.youshitsune.me",
+      "https://searx.zcyph.cc",
+      "https://searxng.tordenskjold.de",
+      "https://searxng.zackptg5.com",
+      "https://serx.ml",
+      "https://swag.pw",
+      "https://www.webrats.xyz",
+      "https://xcxc.ml"
+    ]
+  },
+  {
+    "type": "wikiless",
+    "test_url": "/wiki/Wikipedia?lang=en",
+    "fallback": "https://wikiless.org",
+    "instances": [
+      "https://wiki.604kph.xyz",
+      "https://wiki.froth.zone",
+      "https://wiki.slipfox.xyz",
+      "https://wikiless.esmailelbob.xyz",
+      "https://wikiless.funami.tech",
+      "https://wikiless.lunar.icu",
+      "https://wikiless.northboot.xyz",
+      "https://wikiless.org",
+      "https://wikiless.sethforprivacy.com",
+      "https://wikiless.tiekoetter.com"
+    ]
+  },
+  {
+    "type": "librarian",
+    "test_url": "",
+    "fallback": "https://lbry.bcow.xyz",
+    "instances": [
+      "https://lbry.bcow.xyz",
+      "https://odysee.076.ne.jp",
+      "https://librarian.pussthecat.org",
+      "https://lbry.mutahar.rocks",
+      "https://librarian.esmailelbob.xyz",
+      "https://lbry.vern.cc"
+    ]
+  },
+  {
+    "type": "proxitok",
+    "test_url": "/discover",
+    "fallback": "https://proxitok.herokuapp.com",
+    "instances": [
+      "https://proxitok.herokuapp.com",
+      "https://proxitok.pussthecat.org",
+      "https://proxitok.privacydev.net"
+    ]
+  }
+]

+ 3 - 79
services.json

@@ -4,34 +4,19 @@
     "test_url": "/r/popular",
     "fallback": "https://libredd.it",
     "instances": [
-      "https://libreddit.albony.xyz",
-      "https://libredd.it",
       "https://libreddit.spike.codes",
       "https://libreddit.dothq.co",
-      "https://libreddit.kavin.rocks",
       "https://libreddit.40two.app",
-      "https://reddit.invak.id",
       "https://reddit.phii.me",
-      "https://lr.riverside.rocks",
-      "https://libreddit.silkky.cloud",
-      "https://libreddit.database.red",
-      "https://libreddit.exonip.de",
-      "https://libreddit.domain.glass",
       "https://libreddit.sugoma.tk",
-      "https://libreddit.trevorthalacker.com",
       "https://reddit.artemislena.eu",
-      "https://r.nf",
       "https://libreddit.some-things.org",
       "https://reddit.stuehieyr.com",
       "https://lr.mint.lgbt",
       "https://libreddit.igna.rocks",
       "https://libreddit.autarkic.org",
-      "https://libreddit.flux.industries",
-      "https://libreddit.drivet.xyz",
       "https://libreddit.de",
-      "https://libreddit.bus-hit.me",
-      "https://leddit.xyz",
-      "https://de.leddit.xyz"
+      "https://libreddit.bus-hit.me"
     ]
   },
   {
@@ -39,27 +24,18 @@
     "test_url": "/r/popular",
     "fallback": "https://teddit.net",
     "instances": [
-      "https://i.opnxng.com",
       "https://incogsnoo.com",
       "https://rdt.trom.tf",
       "https://reddit.lol",
       "https://snoo.ioens.is",
-      "https://tedd.it",
       "https://teddit.adminforge.de",
       "https://teddit.bus-hit.me",
-      "https://teddit.domain.glass",
-      "https://teddit.encrypted-data.xyz",
       "https://teddit.froth.zone",
-      "https://teddit.garudalinux.org",
-      "https://teddit.ggc-project.de",
-      "https://teddit.httpjames.space",
-      "https://teddit.namazso.eu",
       "https://teddit.nautolan.racing",
       "https://teddit.net",
       "https://teddit.pussthecat.org",
       "https://teddit.sethforprivacy.com",
       "https://teddit.tinfoil-hat.net",
-      "https://teddit.tokhmi.xyz",
       "https://teddit.totaldarkness.net",
       "https://teddit.zaggy.nl"
     ]
@@ -74,8 +50,7 @@
       "https://bibliogram.pussthecat.org",
       "https://bibliogram.1d4.us",
       "https://insta.trom.tf",
-      "https://bibliogram.hamster.dance",
-      "https://bib.actionsack.com"
+      "https://bibliogram.hamster.dance"
     ]
   },
   {
@@ -84,13 +59,9 @@
     "fallback": "https://invidious.snopyta.org",
     "instances": [
       "https://inv.bp.projectsegfau.lt",
-      "https://inv.privacy.com.de",
-      "https://inv.riverside.rocks",
       "https://inv.vern.cc",
       "https://invidious.esmailelbob.xyz",
       "https://invidious.flokinet.to",
-      "https://invidious.kavin.rocks",
-      "https://invidious.namazso.eu",
       "https://invidious.nerdvpn.de",
       "https://invidious.osi.kr",
       "https://invidious.projectsegfau.lt",
@@ -111,10 +82,6 @@
     "test_url": "/watch?v=eBGIQ7ZuuiU",
     "fallback": "https://piped.kavin.rocks",
     "instances": [
-      "https://piped.kavin.rocks",
-      "https://piped.silkky.cloud",
-      "https://piped.tokhmi.xyz",
-      "https://piped.moomoo.me",
       "https://piped.mint.lgbt"
     ]
   },
@@ -129,26 +96,14 @@
       "https://nitter.nixnet.services",
       "https://nitter.fdn.fr",
       "https://nitter.1d4.us",
-      "https://nitter.kavin.rocks",
       "https://nitter.vxempire.xyz",
       "https://nitter.unixfox.eu",
-      "https://nitter.domain.glass",
       "https://nitter.eu",
-      "https://nitter.namazso.eu",
       "https://nitter.mailstation.de",
-      "https://nitter.actionsack.com",
       "https://nitter.cattube.org",
-      "https://birdsite.xanny.family",
-      "https://nitter.hu",
-      "https://nitter.exonip.de",
       "https://twitr.gq",
-      "https://nitter.moomoo.me",
       "https://bird.trom.tf",
-      "https://nitter.it",
-      "https://twitter.censors.us",
       "https://nitter.grimneko.de",
-      "https://nitter.koyu.space",
-      "https://nitter.ir",
       "https://nitter.autarkic.org",
       "https://n.0x0.st",
       "https://n.hyperborea.cloud",
@@ -156,8 +111,6 @@
       "https://twitter.076.ne.jp",
       "https://nitter.sethforprivacy.com",
       "https://nitter.bus-hit.me",
-      "https://nttr.stream",
-      "https://de.nttr.stream",
       "https://n.l5.ca"
     ]
   },
@@ -183,15 +136,12 @@
       "https://simplytranslate.pussthecat.org",
       "https://st.manerakai.com",
       "https://st.privacydev.net",
-      "https://st.tokhmi.xyz",
       "https://tl.vern.cc",
       "https://translate.beparanoid.de",
       "https://translate.bus-hit.me",
       "https://translate.josias.dev",
-      "https://translate.namazso.eu",
       "https://translate.northboot.xyz",
       "https://translate.priv.pw",
-      "https://translate.riverside.rocks",
       "https://translate.slipfox.xyz",
       "https://translate.tiekoetter.com"
     ]
@@ -213,7 +163,6 @@
     "instances": [
       "https://i.bcow.xyz",
       "https://rimgo.pussthecat.org",
-      "https://img.riverside.rocks",
       "https://rimgo.totaldarkness.net",
       "https://rimgo.bus-hit.me"
     ]
@@ -224,16 +173,9 @@
     "fallback": "https://whoogle.fossho.st",
     "instances": [
       "https://gowogle.voring.me",
-      "https://s.tokhmi.xyz",
-      "https://search.albony.xyz",
-      "https://search.dr460nf1r3.org",
-      "https://search.garudalinux.org",
       "https://search.sethforprivacy.com",
-      "https://search.wef.lol",
-      "https://whoogle.dcs0.hu",
       "https://whoogle.esmailelbob.xyz",
       "https://whoogle.fossho.st",
-      "https://whoogle.lunar.icu",
       "https://whoogle.privacydev.net",
       "https://www.whooglesearch.ml"
     ]
@@ -246,7 +188,6 @@
       "https://dynabyte.ca",
       "https://search.neet.works",
       "https://searx.divided-by-zero.eu",
-      "https://searx.gnu.style",
       "https://searx.rasp.fr",
       "https://searx.ru",
       "https://searx.sp-codes.de",
@@ -254,8 +195,7 @@
       "https://searx.tuxcloud.net",
       "https://searx.tyil.nl",
       "https://searx.webheberg.info",
-      "https://searx.xyz",
-      "https://www.webrats.xyz"
+      "https://searx.xyz"
     ]
   },
   {
@@ -265,13 +205,9 @@
     "instances": [
       "https://darmarit.org/searx",
       "https://de.xcxc.ml",
-      "https://etsi.me",
       "https://jackgoss.xyz",
       "https://northboot.xyz",
-      "https://opnxng.com",
-      "https://paulgo.io",
       "https://priv.au",
-      "https://s.frlt.one",
       "https://s.zhaocloud.net",
       "https://saber.tk",
       "https://search.ashs.club",
@@ -281,12 +217,8 @@
       "https://search.mdosch.de",
       "https://search.neet.works",
       "https://search.ononoki.org",
-      "https://search.privacyguides.net",
       "https://search.projectsegfau.lt",
-      "https://search.rabbit-company.com",
       "https://search.sapti.me",
-      "https://search.teamriverbubbles.com",
-      "https://search.unlocked.link",
       "https://search.zzls.xyz",
       "https://searx.be",
       "https://searx.becomesovran.com",
@@ -294,9 +226,7 @@
       "https://searx.ebnar.xyz",
       "https://searx.ericaftereric.top",
       "https://searx.fmac.xyz",
-      "https://searx.mha.fi",
       "https://searx.mistli.net",
-      "https://searx.namejeff.xyz",
       "https://searx.priv.pw",
       "https://searx.prvcy.eu",
       "https://searx.sev.monster",
@@ -304,12 +234,9 @@
       "https://searx.tiekoetter.com",
       "https://searx.vimproved.me",
       "https://searx.youshitsune.me",
-      "https://searx.zcyph.cc",
       "https://searxng.tordenskjold.de",
-      "https://searxng.zackptg5.com",
       "https://serx.ml",
       "https://swag.pw",
-      "https://www.webrats.xyz",
       "https://xcxc.ml"
     ]
   },
@@ -318,12 +245,9 @@
     "test_url": "/wiki/Wikipedia?lang=en",
     "fallback": "https://wikiless.org",
     "instances": [
-      "https://wiki.604kph.xyz",
       "https://wiki.froth.zone",
       "https://wiki.slipfox.xyz",
       "https://wikiless.esmailelbob.xyz",
-      "https://wikiless.funami.tech",
-      "https://wikiless.lunar.icu",
       "https://wikiless.northboot.xyz",
       "https://wikiless.org",
       "https://wikiless.sethforprivacy.com",