Merge pull request #202 from freedomofpress/split-generate-sign

Split generate and sign steps; add more CI checks

.github/workflows/ci.yml

@@ -11,7 +11,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          apt-get update && apt-get install --yes --no-install-recommends make openssl
+          apt-get update && apt-get install --yes --no-install-recommends make openssl python3 python3-poetry
+          poetry install --no-ansi
       - name: Verify ruleset signature
         run: |
           make verify

.gitignore

@@ -7,7 +7,6 @@ test-key.jwk
 public.pem
 
 # Generated files
-rulesets/default.rulesets
 rulesets/default.rulesets.json
 
 # Byte-compiled / optimized / DLL files

Makefile

@@ -1,6 +1,6 @@
 image := fpf.local/securedrop-https-everywhere-ruleset:$(shell cat latest-rulesets-timestamp)
 
-DEFAULT_GOAL: rules
+DEFAULT_GOAL: help
 
 .PHONY: check-black
 check-black: ## Check Python source code formatting with black
@@ -16,9 +16,19 @@ test-key: ## Generates a test key for development/testing purposes locally.
 	openssl rsa -in key.pem -outform PEM -pubout -out public.pem
 	poetry run python jwk.py > test-key.jwk
 
-.PHONY: rules
-rules: ## Regenerates rulesets in preparation for signing ceremony
-	poetry run ./scripts/generate-and-sign
+.PHONY: generate
+generate: ## Regenerates rulesets in preparation for signing ceremony
+	echo "Generating SecureDrop Onion Name rulesets..."
+	poetry run python3 sddir.py
+	poetry run python3 upstream/merge-rulesets.py --source_dir rulesets
+
+.PHONY: sign
+sign: ## Signs the latest ruleset
+	echo "Preparing rulesets for airgapped signature request..."
+	./upstream/async-request.sh public_release.pem .
+	echo "Updating index for SecureDrop rules..."
+	./update_index.sh
+	echo "Finished. Please review local changes, and commit as appropriate."
 
 .PHONY: serve
 serve: ## Builds Nginx container to serve generated files
@@ -32,6 +42,7 @@ serve: ## Builds Nginx container to serve generated files
 verify: ## Verifies the signature of the latest ruleset. Requires openssl to be installed.
 	@echo "Attempting to verify ruleset signature using openssl."
 	@./scripts/verify
+	@poetry run pytest -v
 
 .PHONY: help
 help:

README.md

@@ -2,7 +2,7 @@
 
 # HTTPS-Everywhere Rulesets for SecureDrop
 
-`securedrop-https-everywhere-ruleset` is used to create a signed HTTPS Everywhere ruleset that maps full-length .onion addresses to user-friendly [onion names](https://securedrop.org/faq/getting-onion-name-your-securedrop/) for some news organizations listed in the [SecureDrop directory](https://securedrop.org/directory/). Any time a new onion name is approved, we add its mapping to our HTTPS Everywhere ruleset and deploy it to https://securedrop.org/https-everywhere/ . Tor Browser automatically includes our ruleset in the default HTTPS Everywhere extension and checks for updates on startup. (Tor Browser will soon switch to checking https://securedrop.org/https-everywhere-2021/ which uses our new release signing key).
+`securedrop-https-everywhere-ruleset` is used to create a signed HTTPS Everywhere ruleset that maps full-length .onion addresses to user-friendly [onion names](https://securedrop.org/faq/getting-onion-name-your-securedrop/) for some news organizations listed in the [SecureDrop directory](https://securedrop.org/directory/). Any time a new onion name is approved, we add its mapping to our HTTPS Everywhere ruleset and deploy it to https://securedrop.org/https-everywhere-2021/ . Tor Browser automatically includes our ruleset in the default HTTPS Everywhere extension and checks for updates on startup.
 
 ## Development
 
@@ -24,13 +24,11 @@ which will create `test-key.jwk` in your current working directory.
 
 2. Add their domain name and the requested URL to the `onboarded.txt` via PR into this repository. We match the domain based on the landing page of the organization, comparing the `netloc` in a URL with structure `scheme://netloc/path;parameters?query#fragment`.
 
-3. Next, generate and sign the update ruleset using the following command (requires signing key, please ping a key holder for assistance):
+3. Next, generate the updated ruleset with `make generate` and review the output.
 
-```
-make rules
-```
+4. Once satisfied, you can sign it with `make sign` (requires signing key, please ping a key holder for assistance).
 
-4. Commit all files generated by the script above and open a Pull Request to this repository. Once the PR is merged, the rulesets will automatically be deployed to production.
+5. Commit all files generated by the script above and open a Pull Request to this repository. Once the PR is merged, the rulesets will automatically be deployed to production.
 
 ## Verifying changes
 

pyproject.toml

@@ -14,6 +14,7 @@ pgpy = ">=0.6.0"
 
 [tool.poetry.group.dev.dependencies]
 black = "*"
+pytest = "^8.3.3"
 
 [tool.black]
 line-length = 100

rulesets/default.rulesets

@@ -0,0 +1 @@
+[{"name":"2600: The Hacker Quarterly","target":["2600.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://2600.securedrop.tor.onion","to":"http://cy6wj77vryhcyh6go576hxycjz4wxlo4s5vevdinkw3armwzty5jozyd.onion"}]},{"name":"ABC","target":["abc.au.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://abc.au.securedrop.tor.onion","to":"http://dqa4zahticcobfq5rmmmbewbdtyiznbl75hu23k4i37y7yfoosrh7mqd.onion"}]},{"name":"Aftenposten AS","target":["aftenposten.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://aftenposten.securedrop.tor.onion","to":"http://tiykfvhb562gheutfnedysnhrxpxoztyszkqyroloyepwzxmxien77id.onion"}]},{"name":"Aftonbladet","target":["aftonbladet.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://aftonbladet.securedrop.tor.onion","to":"http://xm33ge4kupk5o66eqxcd2r4fqcplpqb2sbdduf5z2nw4g2jrxe57luid.onion"}]},{"name":"Al Jazeera Media Network","target":["ajiunit.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://ajiunit.securedrop.tor.onion","to":"http://jkta32w5gvk6pmqdfwj67psojot3l2iwoqbdvrvywi5bkudfeandq7id.onion"}]},{"name":"Apache","target":["apache.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://apache.securedrop.tor.onion","to":"http://okd7utbak43lm7qaixr6yv7s62e32mhngjsfpjn26eklokqofg6776yd.onion"}]},{"name":"Barton Gellman","target":["bartongellman.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://bartongellman.securedrop.tor.onion","to":"http://hxywmnvdz5f2l5gqwjfcejdpla7nhj35dn5cf5l6qevjb77wasnna3qd.onion"}]},{"name":"Bloomberg Industry Group","target":["bloombergindustrygroup.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://bloombergindustrygroup.securedrop.tor.onion","to":"http://33buewrpzrfpttl7kerqvtvzyo3ivumilwwmeqjryzajusltibaqc6ad.onion"}]},{"name":"Bloomberg Law","target":["bloomberglaw.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://bloomberglaw.securedrop.tor.onion","to":"http://33buewrpzrfpttl7kerqvtvzyo3ivumilwwmeqjryzajusltibaqc6ad.onion"}]},{"name":"Bloomberg News","target":["bloomberg.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://bloomberg.securedrop.tor.onion","to":"http://ogdwaroarq4p6rnfn2hl4crvldyruyc2g24435qtxmd3twhevg7dsqid.onion"}]},{"name":"CBC","target":["cbcrc.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://cbcrc.securedrop.tor.onion","to":"http://gppg43zz5d2yfuom3yfmxnnokn3zj4mekt55onlng3zs653ty4fio6qd.onion"}]},{"name":"The Center for Public Integrity","target":["publicintegrity.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://publicintegrity.securedrop.tor.onion","to":"http://ahgpmkiaqfde4innkotgz5q6bgt4gbxmelqod3tjtmpdt3zvxaxareyd.onion"}]},{"name":"Claudio Guarnieri","target":["nex.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://nex.securedrop.tor.onion","to":"http://7dw7foypguycptlodmkscnziw5a65ilivzz6ajiei3yhe3gsfojlqwad.onion"}]},{"name":"CNN","target":["cnn.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://cnn.securedrop.tor.onion","to":"http://qmifwf762qftydprw2adbg7hs2mkunac5xrz3cb5busaflji3rja5lid.onion"}]},{"name":"Dagbladet","target":["dagbladet.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://dagbladet.securedrop.tor.onion","to":"http://ydbpz5knb6ji3bdtahhm3wo7sed6lsy5vqnwfpnhpez4bquvoexbz7qd.onion"}]},{"name":"Der Spiegel","target":["spiegel.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://spiegel.securedrop.tor.onion","to":"http://q6vdlj2ukulrqk37piqgxucpcwtxzdjhvjzqrfbevuhrzimsgjltmpqd.onion"}]},{"name":"Disclose","target":["disclose.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://disclose.securedrop.tor.onion","to":"http://3tcbrdg2ejwu5nzbjg7xqixkis6mdbgkkthcyxmzv2q3oi6v7th5ahqd.onion"}]},{"name":"DR - Danish Broadcasting Corporation","target":["dr.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://dr.securedrop.tor.onion","to":"http://hpaauqmv2wegiu4cz6st6hty4s7gwqol272xhcu3xmh6azw2f2zffgid.onion"}]},{"name":"Espen Andersen","target":["espena.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://espena.securedrop.tor.onion","to":"http://tsovw443sbbaizc3mxwuqrnbc4uiml3x3uuinmplthsmpiqdphl7v5yd.onion"}]},{"name":"Financial Times","target":["ft.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://ft.securedrop.tor.onion","to":"http://nqu6crmtnzs2hs5abo2uqni53yqsnnwqnerdxuzyz5yxairxlzjzt6yd.onion"}]},{"name":"Forbes","target":["forbes.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://forbes.securedrop.tor.onion","to":"http://6zonlfhh7aqtfwoyvdlad3nxn6ljecx2k6tyyy3spt43nn54q6lvncid.onion"}]},{"name":"Forbidden Stories","target":["forbiddenstories.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://forbiddenstories.securedrop.tor.onion","to":"http://fg25fqpu2dnxp24xs3jlcley4hp2inshpzek44q3czkhq3zffoqk26id.onion"}]},{"name":"The Globe and Mail","target":["theglobeandmail.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://theglobeandmail.securedrop.tor.onion","to":"http://a4zum5ydurvljrohxqp2rjjal5kro4ge2q2qizuonf2jubkhcr627gad.onion"}]},{"name":"Greekleaks","target":["greekleaks.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://greekleaks.securedrop.tor.onion","to":"http://jatasaqcoe7lqdpcyxo7vl3e5tdvl5jgmtadfat77i25qdj6z6a4ulad.onion"}]},{"name":"The Guardian","target":["theguardian.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://theguardian.securedrop.tor.onion","to":"http://xp44cagis447k3lpb4wwhcqukix6cgqokbuys24vmxmbzmaq2gjvc2yd.onion"}]},{"name":"HuffPost","target":["huffpost.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://huffpost.securedrop.tor.onion","to":"http://ppw2pmtagxykinex6uubypsommtrcg6ytdh6bcr6agq2wxnrweao4cad.onion"}]},{"name":"Institute for Quantitative Social Science at Harvard University","target":["iqss.harvard.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://iqss.harvard.securedrop.tor.onion","to":"http://5kcyaqagvnrvyan7y5ntzreqsn2msowqlmtoo46qju2pctlbkzzztxqd.onion"}]},{"name":"The Intercept","target":["theintercept.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://theintercept.securedrop.tor.onion","to":"http://lhollo6vzrft3w77mgm67fhfv3fjadmf7oinmafa7tbmupc273oi7kid.onion"}]},{"name":"Investigace.cz","target":["investigace.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://investigace.securedrop.tor.onion","to":"http://e2kkexl7exz6rg7fhl4oftkaeojm7wlbw567hqu2tbrjlixsjjoynzad.onion"}]},{"name":"K-Tipp","target":["ktipp.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://ktipp.securedrop.tor.onion","to":"http://tukldpfzdizsrfyvdljnipmvix2dcb5hmfoemcidkw7bq56wxblk6did.onion"}]},{"name":"Kenneth R. Rosen","target":["kennethrrosen.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://kennethrrosen.securedrop.tor.onion","to":"http://dpsw5tvlh2pccviydqw2cz5tjszd34zcdj322oikydqvgsqwitxup7yd.onion"}]},{"name":"Lessig.law LLC","target":["lessig.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://lessig.securedrop.tor.onion","to":"http://o4nhtigrvss5wktskr5ph5m22ewmhk7nr5at2tac2wdsworcqz62vsqd.onion"}]},{"name":"New York Times","target":["nytimes.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://nytimes.securedrop.tor.onion","to":"https://ej3kv4ebuugcmuwxctx5ic7zxh73rnxt42soi3tdneu2c2em55thufqd.onion"}]},{"name":"News24","target":["news24.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://news24.securedrop.tor.onion","to":"http://uhmj4j5pnwbpmkebfze3qgjmkum465fvok376nxtpku5yvyv5takz6qd.onion"}]},{"name":"NOYB","target":["noyb.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://noyb.securedrop.tor.onion","to":"http://xjc4s5z26i2z5tzjzj3w6jwzuomedzsahq4tccktwdcs6fldt4ojznqd.onion"}]},{"name":"NRK","target":["nrk.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://nrk.securedrop.tor.onion","to":"http://537ztcntpbmspja4mkpxldpsoc46mqlssnsaklqnfw3gnlpj5glcjgid.onion"}]},{"name":"POLITICO","target":["politico.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://politico.securedrop.tor.onion","to":"http://mzi5yynpd6qqq3lnh7vnaojy36v3hcorytsut47zwkguhnorduyxwead.onion"}]},{"name":"ProPublica","target":["propublica.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://propublica.securedrop.tor.onion","to":"http://lvtu6mh6dd6ynqcxtd2mseqfkm7g2iuxvjobbyzpgx2jt427zvd7n3ad.onion"}]},{"name":"San Francisco Chronicle","target":["sfchronicle.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://sfchronicle.securedrop.tor.onion","to":"http://b52gknakgsyqqeq476oi5nymw6yapysfig4owqgwppi5qpuk4az6bxad.onion"}]},{"name":"Stavanger Aftenblad","target":["aftenbladet.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://aftenbladet.securedrop.tor.onion","to":"http://4beybcv5e7xya4xu2nzdqkohawm32imugjtatkvmp2xwgfhcoj64slid.onion"}]},{"name":"Stefania Maurizi","target":["maurizi.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://maurizi.securedrop.tor.onion","to":"http://jxsb4ovmavjy3r64bak4ha63xwggf3nzf3vikvs23r2avm5rhzmaqtqd.onion"}]},{"name":"Suddeutsche Zeitung","target":["sueddeutsche.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://sueddeutsche.securedrop.tor.onion","to":"http://udhauo3m3fh7v6yfiuornjzxn3fh6vlp4ooo3wogvghcnv5xik6mnayd.onion"}]},{"name":"Taz","target":["taz.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://taz.securedrop.tor.onion","to":"http://tazleakssvtc2lqrhkpvbzo6qwolcldzkzoexo7wombufd6a573bhlid.onion"}]},{"name":"TechCrunch","target":["techcrunch.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://techcrunch.securedrop.tor.onion","to":"http://vplxle7awnyvvvduv6exnwrxbf4gzsh7lv7fxosnfl2ecidkttcbfcqd.onion"}]},{"name":"The Economist","target":["theeconomist.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://theeconomist.securedrop.tor.onion","to":"http://mxmddqsh4jnr4gjan37ayin3fu5ecnejxge4wjhj4i45qq5djbxdjtad.onion"}]},{"name":"Thomson Reuters","target":["reuters.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://reuters.securedrop.tor.onion","to":"http://dvvbik7vtmvwwgj2cziqa36noa26l2pweghd26e5l5qwdnqtwmfhz5id.onion"}]},{"name":"Toronto Star","target":["torontostar.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://torontostar.securedrop.tor.onion","to":"http://yj3b7rgmglcocbbvzrwfbo4d6j2aa7thwupra4yqutbd27v3vxcpvgid.onion"}]},{"name":"TV2 Denmark","target":["tv2.dk.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://tv2.dk.securedrop.tor.onion","to":"http://srumyob2jq5nvppzt66aaab333n2wmq6xgkg4khfe24ixdb7umf7mtyd.onion"}]},{"name":"The Washington Post","target":["washingtonpost.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://washingtonpost.securedrop.tor.onion","to":"https://vfnmxpa6fo4jdpyq3yneqhglluweax2uclvxkytfpmpkp5rsl75ir5qd.onion"}]},{"name":"Whistleblower Aid","target":["whistlebloweraid.securedrop.tor.onion"],"rule":[{"from":"^http[s]?://whistlebloweraid.securedrop.tor.onion","to":"http://kogbxf4ysay2qzozmg7ar45ijqmj2vxrwqa4upzqq2i7sqj7wv7wcdqd.onion"}]}]