From 5d70793e2be0dbcd3f4a8b9af6838f8fae9ca4e2 Mon Sep 17 00:00:00 2001 From: ElnuDev Date: Thu, 14 Jul 2022 17:05:45 -0700 Subject: [PATCH] Add support for multiple websites, contents, etc. --- Cargo.lock | 845 +++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 2 + demo/a/index.html | 8 + demo/b/index.html | 8 + demo/index.html | 22 +- demo/soudan.js | 50 ++- demo/style.css | 4 +- src/comment.rs | 3 +- src/database.rs | 20 +- src/main.rs | 150 ++++++-- 10 files changed, 1034 insertions(+), 78 deletions(-) create mode 100644 demo/a/index.html create mode 100644 demo/b/index.html diff --git a/Cargo.lock b/Cargo.lock index e1fa8d8..04b4e16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,13 +58,13 @@ dependencies = [ "http", "httparse", "httpdate", - "itoa", + "itoa 1.0.2", "language-tags", "local-channel", "mime", "percent-encoding", "pin-project-lite", - "rand", + "rand 0.8.5", "sha1", "smallvec", "tracing", @@ -168,7 +168,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "itoa", + "itoa 1.0.2", "language-tags", "log", "mime", @@ -208,7 +208,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom", + "getrandom 0.2.7", "once_cell", "version_check", ] @@ -285,6 +285,18 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bumpalo" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "bytes" version = "1.1.0" @@ -346,6 +358,22 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + [[package]] name = "cpufeatures" version = "0.2.2" @@ -374,6 +402,33 @@ dependencies = [ "typenum", ] +[[package]] +name = "cssparser" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa 0.4.8", + "matches", + "phf 0.8.0", + "proc-macro2", + "quote", + "smallvec", + "syn", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -397,6 +452,27 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dtoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + +[[package]] +name = "dtoa-short" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde03329ae10e79ede66c9ce4dc930aa8599043b0743008548680f25b91502d6" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -418,6 +494,15 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + [[package]] name = "firestorm" version = "0.5.1" @@ -440,6 +525,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -450,6 +550,25 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + +[[package]] +name = "futures-channel" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +dependencies = [ + "futures-core", +] + [[package]] name = "futures-core" version = "0.3.21" @@ -480,6 +599,15 @@ dependencies = [ "pin-utils", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.5" @@ -490,6 +618,26 @@ dependencies = [ "version_check", ] +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.7" @@ -553,6 +701,20 @@ dependencies = [ "libc", ] +[[package]] +name = "html5ever" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "http" version = "0.2.8" @@ -561,7 +723,18 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", - "itoa", + "itoa 1.0.2", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", ] [[package]] @@ -576,6 +749,43 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "hyper" +version = "0.14.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa 1.0.2", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "idna" version = "0.2.3" @@ -603,6 +813,27 @@ dependencies = [ "hashbrown 0.12.1", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "ipnet" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" + +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "itoa" version = "1.0.2" @@ -618,6 +849,15 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "language-tags" version = "0.3.2" @@ -683,6 +923,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" +dependencies = [ + "log", + "phf 0.10.1", + "phf_codegen 0.10.0", + "string_cache", + "string_cache_codegen", + "tendril", +] + [[package]] name = "matches" version = "0.1.9" @@ -728,6 +988,36 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "native-tls" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + [[package]] name = "num-integer" version = "0.1.45" @@ -772,6 +1062,51 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" +[[package]] +name = "openssl" +version = "0.10.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -807,6 +1142,98 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "phf" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +dependencies = [ + "phf_macros", + "phf_shared 0.8.0", + "proc-macro-hack", +] + +[[package]] +name = "phf" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" +dependencies = [ + "phf_shared 0.10.0", +] + +[[package]] +name = "phf_codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" +dependencies = [ + "phf_generator 0.8.0", + "phf_shared 0.8.0", +] + +[[package]] +name = "phf_codegen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", +] + +[[package]] +name = "phf_generator" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" +dependencies = [ + "phf_shared 0.8.0", + "rand 0.7.3", +] + +[[package]] +name = "phf_generator" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +dependencies = [ + "phf_shared 0.10.0", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" +dependencies = [ + "phf_generator 0.8.0", + "phf_shared 0.8.0", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -831,6 +1258,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -855,6 +1288,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + [[package]] name = "proc-macro2" version = "1.0.40" @@ -873,6 +1312,20 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", + "rand_pcg", +] + [[package]] name = "rand" version = "0.8.5" @@ -880,8 +1333,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", ] [[package]] @@ -891,7 +1354,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", ] [[package]] @@ -900,7 +1372,25 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom", + "getrandom 0.2.7", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", ] [[package]] @@ -929,6 +1419,52 @@ version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rusqlite" version = "0.27.0" @@ -960,12 +1496,81 @@ version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" +[[package]] +name = "schannel" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +dependencies = [ + "lazy_static", + "windows-sys", +] + [[package]] name = "scopeguard" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scraper" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5684396b456f3eb69ceeb34d1b5cb1a2f6acf7ca4452131efa3ba0ee2c2d0a70" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "matches", + "selectors", + "smallvec", + "tendril", +] + +[[package]] +name = "security-framework" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "selectors" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "fxhash", + "log", + "matches", + "phf 0.8.0", + "phf_codegen 0.8.0", + "precomputed-hash", + "servo_arc", + "smallvec", + "thin-slice", +] + [[package]] name = "semver" version = "1.0.12" @@ -998,7 +1603,7 @@ version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82c2c1fdcd807d1098552c5b9a36e425e42e9fbd7c6a37a8425f390f781f7fa7" dependencies = [ - "itoa", + "itoa 1.0.2", "ryu", "serde", ] @@ -1010,11 +1615,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa", + "itoa 1.0.2", "ryu", "serde", ] +[[package]] +name = "servo_arc" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" +dependencies = [ + "nodrop", + "stable_deref_trait", +] + [[package]] name = "sha1" version = "0.10.1" @@ -1035,6 +1650,12 @@ dependencies = [ "libc", ] +[[package]] +name = "siphasher" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + [[package]] name = "slab" version = "0.4.6" @@ -1065,12 +1686,46 @@ dependencies = [ "actix-web", "chrono", "md5", + "reqwest", "rusqlite", + "scraper", "serde", "serde_json", "validator", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "string_cache" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08" +dependencies = [ + "new_debug_unreachable", + "once_cell", + "parking_lot", + "phf_shared 0.10.0", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", + "proc-macro2", + "quote", +] + [[package]] name = "syn" version = "1.0.98" @@ -1082,6 +1737,37 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "thin-slice" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" + [[package]] name = "time" version = "0.1.44" @@ -1099,7 +1785,7 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72c91f41dcb2f096c05f0873d667dceec1087ce5bcf984ec8ffb19acddbb3217" dependencies = [ - "itoa", + "itoa 1.0.2", "libc", "num_threads", "time-macros", @@ -1144,6 +1830,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.3" @@ -1158,6 +1854,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.35" @@ -1179,6 +1881,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.15.0" @@ -1206,6 +1914,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + [[package]] name = "url" version = "2.2.2" @@ -1218,6 +1932,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "validator" version = "0.15.0" @@ -1272,6 +1992,22 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -1284,6 +2020,82 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de9a9cec1733468a8c657e57fa2413d2ae2c0129b95e87c5b72b8ace4d13f31f" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" + +[[package]] +name = "web-sys" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1349,6 +2161,15 @@ version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" diff --git a/Cargo.toml b/Cargo.toml index 623bb19..ffe89f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,5 @@ serde_json = "1" validator = { version = "0.15.0", features = ["derive"] } md5 = "0.7.0" chrono = { version = "0.4.19", features = ["serde"] } +reqwest = "0.11.11" +scraper = "0.13.0" diff --git a/demo/a/index.html b/demo/a/index.html new file mode 100644 index 0000000..d7060e8 --- /dev/null +++ b/demo/a/index.html @@ -0,0 +1,8 @@ + + + +

Page A

+

This is Page A.

+
+ + diff --git a/demo/b/index.html b/demo/b/index.html new file mode 100644 index 0000000..e50b5a9 --- /dev/null +++ b/demo/b/index.html @@ -0,0 +1,8 @@ + + + +

Page B

+

This is Page B. +

+ + diff --git a/demo/index.html b/demo/index.html index 3a3aae8..f7c3262 100644 --- a/demo/index.html +++ b/demo/index.html @@ -1,14 +1,10 @@ + - -

Make a comment

-
- - - - - -
-

Comments

-
- - + +

Soudan demo

+

Welcome to the Soudan demo! +

Check out the following example pages, each with their own comment section:

+ diff --git a/demo/soudan.js b/demo/soudan.js index d7bf2cc..bb7c97f 100644 --- a/demo/soudan.js +++ b/demo/soudan.js @@ -1,24 +1,44 @@ -const form = document.getElementById("commentForm"); -const commentContainer = document.getElementById("comments"); -const commentContainerHeader = document.getElementById("commentsHeader"); +document.getElementById("soudan").innerHTML = `

Make a comment

+
+ + + + + +
+

Comments

+
`; +document.write(``); +const url = "http://127.0.0.1:8080"; +const form = document.getElementById("soudan-comment-form"); +const commentContainer = document.getElementById("soudan-comments"); +const commentContainerHeader = document.getElementById("soudan-comments-header"); +const contentId = document.querySelector("meta[name=\"soudan-content-id\"]").getAttribute("content"); form.addEventListener("submit", e => { - let data = {}; + let data = { + url: window.location.href, + comment: { contentId } + }; new FormData(form).forEach((value, key) => { - data[key] = value === "" ? null : value; + data.comment[key] = value === "" ? null : value; }); - var request = new XMLHttpRequest(); - request.open("POST", "http:/127.0.0.1:8080"); - request.send(JSON.stringify(data)); - request.addEventListener("load", () => { - form.querySelector("textarea").value = ""; - reloadComments() - }); - request.addEventListener("error", () => alert("Comment posting failed!")); + fetch(url, { + method: "POST", + body: JSON.stringify(data), + headers: { "Content-Type": "application/json" } + }) + .then(response => { + if (!response.ok) { + return; + } + form.querySelector("textarea").value = ""; + reloadComments(); + }) e.preventDefault(); }); function reloadComments() { - fetch("http://127.0.0.1:8080") + fetch(`${url}/${contentId}`) .then(response => { return response.json().then(json => { return response.ok ? json : Promise.reject(json); @@ -31,7 +51,7 @@ function reloadComments() { html = "

No comments yet! Be the first to make one.

"; } else { comments.forEach(comment => { - html += `
${comment.author ? comment.author : "Anonymous"} commented ${moment(new Date(comment.timestamp * 1000)).fromNow()}:
${comment.text}
`; + html += `
${comment.author ? comment.author : "Anonymous"} commented ${moment(new Date(comment.timestamp * 1000)).fromNow()}:
${comment.text}
`; }); } commentContainer.innerHTML = html; diff --git a/demo/style.css b/demo/style.css index c248493..598e37a 100644 --- a/demo/style.css +++ b/demo/style.css @@ -1,8 +1,8 @@ -#comments > div { +#soudan-comments > div { display: flex; gap: 0.5em; } -#comments .avatar { +.soudan-avatar { border-radius: 100%; width: 80px; height: 80px; diff --git a/src/comment.rs b/src/comment.rs index a3de1a6..6eb16a0 100644 --- a/src/comment.rs +++ b/src/comment.rs @@ -3,8 +3,8 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize, Serializer}; use validator::Validate; -// Master comment type that is stored in database #[derive(Serialize, Deserialize, Validate)] +#[serde(rename_all = "camelCase")] pub struct Comment { #[serde(skip_serializing_if = "Option::is_none")] pub author: Option, // None is Anonymous @@ -19,6 +19,7 @@ pub struct Comment { #[serde(with = "ts_seconds_option")] #[serde(skip_serializing_if = "Option::is_none")] pub timestamp: Option>, + pub content_id: String, } fn serialize_gravatar(email: &Option, s: S) -> Result diff --git a/src/database.rs b/src/database.rs index 8adcc99..b173457 100644 --- a/src/database.rs +++ b/src/database.rs @@ -13,26 +13,28 @@ impl Database { let conn = Connection::open_in_memory()?; conn.execute( "CREATE TABLE comment ( - id INTEGER PRIMARY KEY, - email TEXT, - author TEXT, - text TEXT NOT NULL, - timestamp DATETIME DEFAULT CURRENT_TIMESTAMP + id INTEGER PRIMARY KEY, + email TEXT, + author TEXT, + text TEXT NOT NULL, + timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, + content_id TEXT NOT NULL )", params![], )?; Ok(Self { conn }) } - pub fn get_comments(&self) -> Result> { + pub fn get_comments(&self, content_id: &str) -> Result> { self.conn - .prepare("SELECT author, email, text, timestamp FROM comment ORDER BY timestamp DESC")? + .prepare(&format!("SELECT author, email, text, timestamp FROM comment WHERE content_id='{content_id}' ORDER BY timestamp DESC"))? .query_map([], |row| { Ok(Comment { author: row.get(0)?, email: row.get(1)?, text: row.get(2)?, timestamp: row.get(3)?, + content_id: content_id.to_owned(), }) })? .collect() @@ -40,8 +42,8 @@ impl Database { pub fn create_comment(&self, comment: &Comment) -> Result<()> { self.conn.execute( - "INSERT INTO comment (author, email, text) VALUES (?1, ?2, ?3)", - params![&comment.author, &comment.email, &comment.text], + "INSERT INTO comment (author, email, text, content_id) VALUES (?1, ?2, ?3, ?4)", + params![&comment.author, &comment.email, &comment.text, &comment.content_id], )?; Ok(()) } diff --git a/src/main.rs b/src/main.rs index 381016d..c63e58f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,63 +1,161 @@ -mod comment; -use actix_cors::Cors; +mod comment; use actix_cors::Cors; pub use comment::*; mod database; pub use database::Database; -use actix_web::{get, post, web, App, HttpResponse, HttpServer, Responder}; -use std::{env, sync::Mutex}; +use actix_web::{get, post, web, App, HttpResponse, HttpRequest, HttpServer, Responder}; +use std::{env, sync::{Mutex, MutexGuard}}; use validator::Validate; +use scraper::{Html, Selector}; +use std::collections::HashMap; +use serde::Deserialize; struct AppState { - db: Mutex, + databases: HashMap>, } -#[get("/")] -async fn get_comments(data: web::Data) -> impl Responder { - let db = match data.db.lock() { - Ok(db) => db, - Err(_) => return HttpResponse::InternalServerError().into(), +fn get_db<'a>(data: &'a web::Data, request: &HttpRequest) -> Result, HttpResponse> { + // all the .into() are converting from HttpResponseBuilder to HttpResponse + let origin = match request.head().headers().get("Origin") { + Some(origin) => match origin.to_str() { + Ok(origin) => origin, + Err(_) => return Err(HttpResponse::BadRequest().into()), + } + None => return Err(HttpResponse::BadRequest().into()), + }; + match data.databases.get(origin) { + Some(database) => Ok(match database.lock() { + Ok(database) => database, + Err(_) => return Err(HttpResponse::InternalServerError().into()), + }), + None => return Err(HttpResponse::BadRequest().into()), + } +} + +#[get("/{content_id}")] +async fn get_comments(data: web::Data, request: HttpRequest, content_id: web::Path) -> impl Responder { + let database = match get_db(&data, &request) { + Ok(database) => database, + Err(response) => return response, }; - HttpResponse::Ok().json(&db.get_comments().unwrap()) + HttpResponse::Ok().json(database.get_comments(&content_id).unwrap()) +} + +#[derive(Deserialize)] +struct PostCommentsRequest { + url: String, + comment: Comment, } #[post("/")] -async fn post_comment(data: web::Data, bytes: web::Bytes) -> impl Responder { +async fn post_comment(data: web::Data, request: HttpRequest, bytes: web::Bytes) -> impl Responder { match String::from_utf8(bytes.to_vec()) { Ok(text) => { - let db = match data.db.lock() { - Ok(db) => db, - Err(_) => return HttpResponse::InternalServerError(), - }; - let comment: Comment = match serde_json::from_str(&text) { - Ok(comment) => comment, - Err(_) => return HttpResponse::BadRequest(), + let PostCommentsRequest { url, comment } = match serde_json::from_str(&text) { + Ok(req) => req, + Err(_) => return HttpResponse::BadRequest().into(), }; if comment.validate().is_err() { - return HttpResponse::BadRequest(); + return HttpResponse::BadRequest().into(); + } + let origin = match request.head().headers().get("Origin") { + Some(origin) => match origin.to_str() { + Ok(origin) => origin, + // If the Origin is not valid ASCII, it is a bad request not sent from a browser + Err(_) => return HttpResponse::BadRequest().into(), + }, + // If there is no Origin header, it is a bad request not sent from a browser + None => return HttpResponse::BadRequest().into(), + }; + // Check to see if provided URL is in scope. + // This is to prevent malicious requests that try to get server to fetch external websites. + // (requires loop because "labels on blocks are unstable") + // https://github.com/rust-lang/rust/issues/48594 + 'outer: loop { + for site_root in data.databases.keys() { + if site_root.starts_with(origin) && url.starts_with(site_root) { + break 'outer; + } + } + return HttpResponse::BadRequest().into(); } - db.create_comment(&comment).unwrap(); - HttpResponse::Ok() + match get_page_data(&url).await { + Ok(page_data_option) => match page_data_option { + Some(page_data) => if page_data.content_id != comment.content_id { + return HttpResponse::BadRequest().into(); + }, + None => return HttpResponse::BadRequest().into(), + }, + Err(_) => return HttpResponse::InternalServerError().into(), + }; + let database = match get_db(&data, &request) { + Ok(database) => database, + Err(response) => return response, + }; + database.create_comment(&comment).unwrap(); + HttpResponse::Ok().into() } Err(_) => HttpResponse::BadRequest().into(), } } +// Contains all page details stored in meta tags. +// Currently, only content_id, but this is wrapped in this struct +// to make adding other meta tags, such as locked comments, in the future +struct PageData { + content_id: String, +} + +async fn get_page_data(url: &str) -> Result, reqwest::Error> { + let response = reqwest::get(url).await?; + if !response.status().is_success() { + return Ok(None); + } + let content = response.text_with_charset("utf-8").await?; + let document = Html::parse_document(&content); + let get_meta = |name: &str| -> Option { + let selector = Selector::parse(&format!("meta[name=\"{}\"]", name)).unwrap(); + match document.select(&selector).next() { + Some(element) => match element.value().attr("content") { + Some(value) => Some(value.to_owned()), + None => return None, + }, + None => return None, + } + }; + return Ok(Some(PageData { + content_id: match get_meta("soudan-content-id") { + Some(id) => id, + None => return Ok(None), + }, + })) +} + #[actix_web::main] async fn main() -> Result<(), std::io::Error> { + let mut domains = Vec::new(); let testing = { let mut testing = false; - for argument in env::args() { + let mut args = env::args(); + args.next(); // Skip first, will be executable name + for argument in args { if argument == "--testing" || argument == "-t" { testing = true; - break; + } else { + domains.push(argument); } } testing }; - let db = Database::new(testing).unwrap(); - let state = web::Data::new(AppState { db: Mutex::new(db) }); + if domains.len() == 0 { + panic!("At least one domain is required!"); + } + let mut databases = HashMap::new(); + for domain in domains.iter() { + databases.insert(domain.to_owned(), Mutex::new(Database::new(testing).unwrap())); + } + let state = web::Data::new(AppState { databases }); HttpServer::new(move || { App::new() .service(get_comments)