From e1e46dcd9271e68492db5fe259c1593f02e7ef84 Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Thu, 4 Aug 2022 13:16:58 -0600 Subject: [PATCH] Remove cloudflare filter from update workflow The cloudflare filter, when performed as part of the github action workflow, doesn't seem to work nearly as reliably when run on an actual machine. The farside server will instead run the un-cloudflare script whenever it pulls in new changes to services-full.json, which should be a much more reliable approach to filtering out cloudflare instances. --- .github/workflows/update-instances.yml | 32 ----------- services.json | 73 +++++--------------------- tools/un-cloudflare.sh | 4 +- 3 files changed, 14 insertions(+), 95 deletions(-) diff --git a/.github/workflows/update-instances.yml b/.github/workflows/update-instances.yml index 17da660..7f483ac 100644 --- a/.github/workflows/update-instances.yml +++ b/.github/workflows/update-instances.yml @@ -27,33 +27,6 @@ jobs: sed -i 's/\/"/"/g' services-full.json } - function remove_cf_instances() { - rm -f out.json - file="services-full.json" - - while read -r line; do - if [[ "$line" == "\"https://"* ]]; then - domain=$(echo "$line" | sed -e "s/^\"https:\/\///" -e "s/\",//" -e "s/\"//") - ns=$(dig ns "$domain" || true) - if [[ "$ns" == *"cloudflare"* ]]; then - echo "\"$domain\" using cloudflare, skipping..." - elif [[ "$ns" != *"NOERROR"* ]]; then - echo "Unable to verify records for \"$domain\", skipping..." - else - echo "$line" >> out.json - fi - else - echo "$line" >> out.json - fi - done <$file - - # Remove any trailing commas from new instance lists - sed -i -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json - - cat out.json | jq --indent 2 . > services.json - rm -f out.json - } - # ============================================================== # Git config # ============================================================== @@ -216,11 +189,6 @@ jobs: # TODO: Update instances for other services # ============================================================== - # ============================================================== - # Remove cloudflare instances - # ============================================================== - remove_cf_instances - # ============================================================== # Push changes # ============================================================== diff --git a/services.json b/services.json index 4300fdf..ea5ce7d 100644 --- a/services.json +++ b/services.json @@ -4,29 +4,19 @@ "test_url": "/r/popular", "fallback": "https://libredd.it", "instances": [ - "https://libreddit.albony.xyz", "https://libreddit.spike.codes", "https://libreddit.dothq.co", - "https://libreddit.kavin.rocks", "https://libreddit.40two.app", - "https://reddit.invak.id", - "https://lr.riverside.rocks", - "https://libreddit.silkky.cloud", - "https://libreddit.database.red", - "https://libreddit.exonip.de", - "https://libreddit.domain.glass", - "https://libreddit.trevorthalacker.com", + "https://reddit.phii.me", + "https://libreddit.sugoma.tk", "https://reddit.artemislena.eu", "https://libreddit.some-things.org", "https://reddit.stuehieyr.com", - "https://lr.mint.lgbt", "https://libreddit.igna.rocks", "https://libreddit.autarkic.org", "https://libreddit.flux.industries", - "https://libreddit.drivet.xyz", "https://libreddit.de", - "https://libreddit.bus-hit.me", - "https://de.leddit.xyz" + "https://libreddit.bus-hit.me" ] }, { @@ -37,18 +27,15 @@ "https://incogsnoo.com", "https://rdt.trom.tf", "https://reddit.lol", + "https://snoo.ioens.is", "https://teddit.adminforge.de", "https://teddit.bus-hit.me", - "https://teddit.domain.glass", - "https://teddit.encrypted-data.xyz", "https://teddit.froth.zone", - "https://teddit.garudalinux.org", - "https://teddit.ggc-project.de", - "https://teddit.httpjames.space", - "https://teddit.namazso.eu", + "https://teddit.nautolan.racing", "https://teddit.net", "https://teddit.pussthecat.org", "https://teddit.sethforprivacy.com", + "https://teddit.tinfoil-hat.net", "https://teddit.totaldarkness.net", "https://teddit.zaggy.nl" ] @@ -58,7 +45,6 @@ "test_url": "/u/officialrickastley", "fallback": "https://bibliogram.art", "instances": [ - "https://bib.actionsack.com", "https://bibliogram.1d4.us", "https://bibliogram.art", "https://bibliogram.froth.zone", @@ -67,8 +53,6 @@ "https://bibliogram.pussthecat.org", "https://bibliogram.snopyta.org", "https://ig.beparanoid.de", - "https://ig.funami.tech", - "https://ig.tokhmi.xyz", "https://insta.trom.tf", "https://insta.tromdienste.de" ] @@ -79,13 +63,9 @@ "fallback": "https://invidious.snopyta.org", "instances": [ "https://inv.bp.projectsegfau.lt", - "https://inv.privacy.com.de", - "https://inv.riverside.rocks", "https://inv.vern.cc", "https://invidious.esmailelbob.xyz", "https://invidious.flokinet.to", - "https://invidious.kavin.rocks", - "https://invidious.namazso.eu", "https://invidious.nerdvpn.de", "https://invidious.osi.kr", "https://invidious.projectsegfau.lt", @@ -106,10 +86,6 @@ "test_url": "/watch?v=eBGIQ7ZuuiU", "fallback": "https://piped.kavin.rocks", "instances": [ - "https://piped.kavin.rocks", - "https://piped.silkky.cloud", - "https://piped.tokhmi.xyz", - "https://piped.moomoo.me", "https://piped.mint.lgbt" ] }, @@ -124,29 +100,21 @@ "https://nitter.nixnet.services", "https://nitter.fdn.fr", "https://nitter.1d4.us", - "https://nitter.kavin.rocks", "https://nitter.vxempire.xyz", "https://nitter.unixfox.eu", "https://nitter.eu", - "https://nitter.namazso.eu", "https://nitter.mailstation.de", - "https://n.actionsack.com", "https://nitter.cattube.org", - "https://nitter.exonip.de", "https://twitr.gq", - "https://nitter.moomoo.me", "https://bird.trom.tf", "https://nitter.it", - "https://twitter.censors.us", "https://nitter.grimneko.de", - "https://nitter.koyu.space", "https://n.0x0.st", "https://n.hyperborea.cloud", "https://nitter.ca", "https://twitter.076.ne.jp", "https://nitter.sethforprivacy.com", "https://nitter.bus-hit.me", - "https://de.nttr.stream", "https://n.l5.ca", "https://unofficialbird.com" ] @@ -177,10 +145,9 @@ "https://tl.vern.cc", "https://translate.beparanoid.de", "https://translate.bus-hit.me", - "https://translate.namazso.eu", + "https://translate.josias.dev", "https://translate.northboot.xyz", "https://translate.priv.pw", - "https://translate.riverside.rocks", "https://translate.slipfox.xyz", "https://translate.tiekoetter.com" ] @@ -202,7 +169,6 @@ "instances": [ "https://i.bcow.xyz", "https://rimgo.pussthecat.org", - "https://img.riverside.rocks", "https://rimgo.totaldarkness.net", "https://rimgo.bus-hit.me" ] @@ -213,15 +179,10 @@ "fallback": "https://whoogle.fossho.st", "instances": [ "https://gowogle.voring.me", - "https://s.tokhmi.xyz", - "https://search.albony.xyz", - "https://search.dr460nf1r3.org", - "https://search.garudalinux.org", "https://search.sethforprivacy.com", - "https://search.wef.lol", - "https://whoogle.dcs0.hu", "https://whoogle.esmailelbob.xyz", "https://whoogle.fossho.st", + "https://whoogle.lunar.icu", "https://whoogle.privacydev.net", "https://www.whooglesearch.ml" ] @@ -235,7 +196,6 @@ "https://dynabyte.ca", "https://search.neet.works", "https://searx.divided-by-zero.eu", - "https://searx.gnu.style", "https://searx.ru", "https://searx.sp-codes.de", "https://searx.stuehieyr.com", @@ -249,10 +209,10 @@ "test_url": "/search?q=<%=query%>", "fallback": "https://searx.be", "instances": [ + "https://darmarit.org/searx", "https://de.xcxc.ml", "https://jackgoss.xyz", "https://priv.au", - "https://s.frlt.one", "https://s.zhaocloud.net", "https://saber.tk", "https://search.0relay.com", @@ -263,12 +223,8 @@ "https://search.mdosch.de", "https://search.neet.works", "https://search.ononoki.org", - "https://search.privacyguides.net", "https://search.projectsegfau.lt", - "https://search.rabbit-company.com", "https://search.sapti.me", - "https://search.teamriverbubbles.com", - "https://search.unlocked.link", "https://search.zzls.xyz", "https://searx.be", "https://searx.becomesovran.com", @@ -278,14 +234,12 @@ "https://searx.fmac.xyz", "https://searx.mha.fi", "https://searx.mistli.net", - "https://searx.namejeff.xyz", "https://searx.priv.pw", "https://searx.prvcy.eu", "https://searx.sev.monster", + "https://searx.slipfox.xyz/searx", "https://searx.tiekoetter.com", - "https://searx.zcyph.cc", "https://searxng.tordenskjold.de", - "https://searxng.zackptg5.com", "https://serx.ml", "https://swag.pw", "https://xcxc.ml" @@ -296,11 +250,10 @@ "test_url": "/wiki/Wikipedia?lang=en", "fallback": "https://wikiless.org", "instances": [ - "https://wiki.604kph.xyz", "https://wiki.froth.zone", "https://wiki.slipfox.xyz", "https://wikiless.esmailelbob.xyz", - "https://wikiless.funami.tech", + "https://wikiless.lunar.icu", "https://wikiless.northboot.xyz", "https://wikiless.org", "https://wikiless.sethforprivacy.com", @@ -336,9 +289,7 @@ "fallback": "https://librex.beparanoid.de", "instances": [ "https://librex.beparanoid.de", - "https://search.davidovski.xyz", - "https://search.funami.tech", - "https://librex.catalyst.sx" + "https://search.davidovski.xyz" ] }, { diff --git a/tools/un-cloudflare.sh b/tools/un-cloudflare.sh index b10178b..dd5c716 100755 --- a/tools/un-cloudflare.sh +++ b/tools/un-cloudflare.sh @@ -10,7 +10,7 @@ while read -r line; do ns=$(dig ns "$domain" || true) if [[ "$ns" == *"cloudflare"* ]]; then echo "\"$domain\" using cloudflare, skipping..." - elif [ ${#ns} -eq 0 ]; then + elif [[ "$ns" != *"NOERROR"* ]]; then echo "Unable to verify records for \"$domain\", skipping..." else echo "$line" >> out.json @@ -21,7 +21,7 @@ while read -r line; do done <$file # Remove any trailing commas from new instance lists -sed -i '' -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json +sed -i -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json cat out.json | jq --indent 2 . > services.json rm -f out.json