Remove cloudflare filter from update workflow

The cloudflare filter, when performed as part of the github action
workflow, doesn't seem to work nearly as reliably when run on an actual
machine.

The farside server will instead run the un-cloudflare script whenever it
pulls in new changes to services-full.json, which should be a much more
reliable approach to filtering out cloudflare instances.

.github/workflows/update-instances.yml

@@ -27,33 +27,6 @@ jobs:
           sed -i 's/\/"/"/g' services-full.json
         }
 
-        function remove_cf_instances() {
-          rm -f out.json
-          file="services-full.json"
-
-          while read -r line; do
-            if [[ "$line" == "\"https://"* ]]; then
-              domain=$(echo "$line" | sed -e "s/^\"https:\/\///" -e "s/\",//" -e "s/\"//")
-              ns=$(dig ns "$domain" || true)
-              if [[ "$ns" == *"cloudflare"* ]]; then
-                echo "\"$domain\" using cloudflare, skipping..."
-              elif [[ "$ns" != *"NOERROR"* ]]; then
-                echo "Unable to verify records for \"$domain\", skipping..."
-              else
-                echo "$line" >> out.json
-              fi
-            else
-              echo "$line" >> out.json
-            fi
-          done <$file
-
-          # Remove any trailing commas from new instance lists
-          sed -i -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json
-
-          cat out.json | jq --indent 2 . > services.json
-          rm -f out.json
-        }
-
         # ==============================================================
         # Git config
         # ==============================================================
@@ -216,11 +189,6 @@ jobs:
         # TODO: Update instances for other services
         # ==============================================================
 
-        # ==============================================================
-        # Remove cloudflare instances
-        # ==============================================================
-        remove_cf_instances
-
         # ==============================================================
         # Push changes
         # ==============================================================

services.json

@@ -4,29 +4,19 @@
     "test_url": "/r/popular",
     "fallback": "https://libredd.it",
     "instances": [
-      "https://libreddit.albony.xyz",
       "https://libreddit.spike.codes",
       "https://libreddit.dothq.co",
-      "https://libreddit.kavin.rocks",
       "https://libreddit.40two.app",
-      "https://reddit.invak.id",
+      "https://reddit.phii.me",
-      "https://lr.riverside.rocks",
+      "https://libreddit.sugoma.tk",
-      "https://libreddit.silkky.cloud",
-      "https://libreddit.database.red",
-      "https://libreddit.exonip.de",
-      "https://libreddit.domain.glass",
-      "https://libreddit.trevorthalacker.com",
       "https://reddit.artemislena.eu",
       "https://libreddit.some-things.org",
       "https://reddit.stuehieyr.com",
-      "https://lr.mint.lgbt",
       "https://libreddit.igna.rocks",
       "https://libreddit.autarkic.org",
       "https://libreddit.flux.industries",
-      "https://libreddit.drivet.xyz",
       "https://libreddit.de",
-      "https://libreddit.bus-hit.me",
+      "https://libreddit.bus-hit.me"
-      "https://de.leddit.xyz"
     ]
   },
   {
@@ -37,18 +27,15 @@
       "https://incogsnoo.com",
       "https://rdt.trom.tf",
       "https://reddit.lol",
+      "https://snoo.ioens.is",
       "https://teddit.adminforge.de",
       "https://teddit.bus-hit.me",
-      "https://teddit.domain.glass",
-      "https://teddit.encrypted-data.xyz",
       "https://teddit.froth.zone",
-      "https://teddit.garudalinux.org",
+      "https://teddit.nautolan.racing",
-      "https://teddit.ggc-project.de",
-      "https://teddit.httpjames.space",
-      "https://teddit.namazso.eu",
       "https://teddit.net",
       "https://teddit.pussthecat.org",
       "https://teddit.sethforprivacy.com",
+      "https://teddit.tinfoil-hat.net",
       "https://teddit.totaldarkness.net",
       "https://teddit.zaggy.nl"
     ]
@@ -58,7 +45,6 @@
     "test_url": "/u/officialrickastley",
     "fallback": "https://bibliogram.art",
     "instances": [
-      "https://bib.actionsack.com",
       "https://bibliogram.1d4.us",
       "https://bibliogram.art",
       "https://bibliogram.froth.zone",
@@ -67,8 +53,6 @@
       "https://bibliogram.pussthecat.org",
       "https://bibliogram.snopyta.org",
       "https://ig.beparanoid.de",
-      "https://ig.funami.tech",
-      "https://ig.tokhmi.xyz",
       "https://insta.trom.tf",
       "https://insta.tromdienste.de"
     ]
@@ -79,13 +63,9 @@
     "fallback": "https://invidious.snopyta.org",
     "instances": [
       "https://inv.bp.projectsegfau.lt",
-      "https://inv.privacy.com.de",
-      "https://inv.riverside.rocks",
       "https://inv.vern.cc",
       "https://invidious.esmailelbob.xyz",
       "https://invidious.flokinet.to",
-      "https://invidious.kavin.rocks",
-      "https://invidious.namazso.eu",
       "https://invidious.nerdvpn.de",
       "https://invidious.osi.kr",
       "https://invidious.projectsegfau.lt",
@@ -106,10 +86,6 @@
     "test_url": "/watch?v=eBGIQ7ZuuiU",
     "fallback": "https://piped.kavin.rocks",
     "instances": [
-      "https://piped.kavin.rocks",
-      "https://piped.silkky.cloud",
-      "https://piped.tokhmi.xyz",
-      "https://piped.moomoo.me",
       "https://piped.mint.lgbt"
     ]
   },
@@ -124,29 +100,21 @@
       "https://nitter.nixnet.services",
       "https://nitter.fdn.fr",
       "https://nitter.1d4.us",
-      "https://nitter.kavin.rocks",
       "https://nitter.vxempire.xyz",
       "https://nitter.unixfox.eu",
       "https://nitter.eu",
-      "https://nitter.namazso.eu",
       "https://nitter.mailstation.de",
-      "https://n.actionsack.com",
       "https://nitter.cattube.org",
-      "https://nitter.exonip.de",
       "https://twitr.gq",
-      "https://nitter.moomoo.me",
       "https://bird.trom.tf",
       "https://nitter.it",
-      "https://twitter.censors.us",
       "https://nitter.grimneko.de",
-      "https://nitter.koyu.space",
       "https://n.0x0.st",
       "https://n.hyperborea.cloud",
       "https://nitter.ca",
       "https://twitter.076.ne.jp",
       "https://nitter.sethforprivacy.com",
       "https://nitter.bus-hit.me",
-      "https://de.nttr.stream",
       "https://n.l5.ca",
       "https://unofficialbird.com"
     ]
@@ -177,10 +145,9 @@
       "https://tl.vern.cc",
       "https://translate.beparanoid.de",
       "https://translate.bus-hit.me",
-      "https://translate.namazso.eu",
+      "https://translate.josias.dev",
       "https://translate.northboot.xyz",
       "https://translate.priv.pw",
-      "https://translate.riverside.rocks",
       "https://translate.slipfox.xyz",
       "https://translate.tiekoetter.com"
     ]
@@ -202,7 +169,6 @@
     "instances": [
       "https://i.bcow.xyz",
       "https://rimgo.pussthecat.org",
-      "https://img.riverside.rocks",
       "https://rimgo.totaldarkness.net",
       "https://rimgo.bus-hit.me"
     ]
@@ -213,15 +179,10 @@
     "fallback": "https://whoogle.fossho.st",
     "instances": [
       "https://gowogle.voring.me",
-      "https://s.tokhmi.xyz",
-      "https://search.albony.xyz",
-      "https://search.dr460nf1r3.org",
-      "https://search.garudalinux.org",
       "https://search.sethforprivacy.com",
-      "https://search.wef.lol",
-      "https://whoogle.dcs0.hu",
       "https://whoogle.esmailelbob.xyz",
       "https://whoogle.fossho.st",
+      "https://whoogle.lunar.icu",
       "https://whoogle.privacydev.net",
       "https://www.whooglesearch.ml"
     ]
@@ -235,7 +196,6 @@
       "https://dynabyte.ca",
       "https://search.neet.works",
       "https://searx.divided-by-zero.eu",
-      "https://searx.gnu.style",
       "https://searx.ru",
       "https://searx.sp-codes.de",
       "https://searx.stuehieyr.com",
@@ -249,10 +209,10 @@
     "test_url": "/search?q=<%=query%>",
     "fallback": "https://searx.be",
     "instances": [
+      "https://darmarit.org/searx",
       "https://de.xcxc.ml",
       "https://jackgoss.xyz",
       "https://priv.au",
-      "https://s.frlt.one",
       "https://s.zhaocloud.net",
       "https://saber.tk",
       "https://search.0relay.com",
@@ -263,12 +223,8 @@
       "https://search.mdosch.de",
       "https://search.neet.works",
       "https://search.ononoki.org",
-      "https://search.privacyguides.net",
       "https://search.projectsegfau.lt",
-      "https://search.rabbit-company.com",
       "https://search.sapti.me",
-      "https://search.teamriverbubbles.com",
-      "https://search.unlocked.link",
       "https://search.zzls.xyz",
       "https://searx.be",
       "https://searx.becomesovran.com",
@@ -278,14 +234,12 @@
       "https://searx.fmac.xyz",
       "https://searx.mha.fi",
       "https://searx.mistli.net",
-      "https://searx.namejeff.xyz",
       "https://searx.priv.pw",
       "https://searx.prvcy.eu",
       "https://searx.sev.monster",
+      "https://searx.slipfox.xyz/searx",
       "https://searx.tiekoetter.com",
-      "https://searx.zcyph.cc",
       "https://searxng.tordenskjold.de",
-      "https://searxng.zackptg5.com",
       "https://serx.ml",
       "https://swag.pw",
       "https://xcxc.ml"
@@ -296,11 +250,10 @@
     "test_url": "/wiki/Wikipedia?lang=en",
     "fallback": "https://wikiless.org",
     "instances": [
-      "https://wiki.604kph.xyz",
       "https://wiki.froth.zone",
       "https://wiki.slipfox.xyz",
       "https://wikiless.esmailelbob.xyz",
-      "https://wikiless.funami.tech",
+      "https://wikiless.lunar.icu",
       "https://wikiless.northboot.xyz",
       "https://wikiless.org",
       "https://wikiless.sethforprivacy.com",
@@ -336,9 +289,7 @@
     "fallback": "https://librex.beparanoid.de",
     "instances": [
       "https://librex.beparanoid.de",
-      "https://search.davidovski.xyz",
+      "https://search.davidovski.xyz"
-      "https://search.funami.tech",
-      "https://librex.catalyst.sx"
     ]
   },
   {

tools/un-cloudflare.sh

@@ -10,7 +10,7 @@ while read -r line; do
         ns=$(dig ns "$domain" || true)
         if [[ "$ns" == *"cloudflare"* ]]; then
             echo "\"$domain\" using cloudflare, skipping..."
-        elif [ ${#ns} -eq 0 ]; then
+        elif [[ "$ns" != *"NOERROR"* ]]; then
             echo "Unable to verify records for \"$domain\", skipping..."
         else
             echo "$line" >> out.json
@@ -21,7 +21,7 @@ while read -r line; do
 done <$file
 
 # Remove any trailing commas from new instance lists
-sed -i '' -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json
+sed -i -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json
 
 cat out.json | jq --indent 2 . > services.json
 rm -f out.json