diff --git a/Cargo.lock b/Cargo.lock index 19ff500..ed1ce6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1120,6 +1120,16 @@ dependencies = [ "encoding", ] +[[package]] +name = "gh-emoji" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ad64b43d48c1745c0059e5ba223816eb599eec8956c668fc0a31f6b9cd799e" +dependencies = [ + "phf", + "regex", +] + [[package]] name = "ghash" version = "0.5.0" @@ -2502,9 +2512,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" dependencies = [ "aho-corasick 1.0.1", "memchr", @@ -3276,9 +3286,11 @@ dependencies = [ "derive_more", "dotenv", "gettext", + "gh-emoji", "poise", "r2d2", "r2d2_sqlite", + "regex", "reqwest", "rocket 0.5.0-rc.3", "rocket_contrib", diff --git a/Cargo.toml b/Cargo.toml index c068c0f..856106b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,9 +13,11 @@ comrak = "0.18.0" derive_more = "0.99.17" dotenv = "0.15.0" gettext = "0.4.0" +gh-emoji = "1.0.7" poise = "0.5.5" r2d2 = "0.8.10" r2d2_sqlite = "0.22.0" +regex = "1.8.4" reqwest = "0.11.18" rocket = { version = "=0.5.0-rc.3", features = ["secrets", "json"] } rocket_contrib = { version = "0.4.11", features = ["templates"] } diff --git a/content/challenges/110.md b/content/challenges/110.md index 550289a..2c5b872 100644 --- a/content/challenges/110.md +++ b/content/challenges/110.md @@ -17,7 +17,6 @@ japanese: - 霊: れい - pos: particle text: は - - pos: particle - pos: verb text: - 踊: おど diff --git a/content/challenges/66.md b/content/challenges/66.md index 1988bb4..e6459f9 100644 --- a/content/challenges/66.md +++ b/content/challenges/66.md @@ -1,8 +1,8 @@ --- japanese: - - - - pos: noun + - - - pos: verb text: スイーツ - pos: verb + - pos: verb text: - 食: た - べ @@ -89,3 +89,5 @@ youtube: IqZ2qcE3XVc suggester: 猛火妹紅（MochaMoko） date: 2022-06-17 --- + +**Note 2023-07-01:** The first word of the first line, スイーツ, was missing in the original challenge and has been corrected. \ No newline at end of file diff --git a/content/challenges/99.md b/content/challenges/99.md index dc8cd87..65c578a 100644 --- a/content/challenges/99.md +++ b/content/challenges/99.md @@ -101,4 +101,4 @@ suggester: モカ妹紅（MochaMoko） date: 2023-02-14 --- -{{< banner >}}:love_letter: Happy Valentine's Day!{{< /banner >}} +:love_letter: Happy Valentine's Day! \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index aaec564..0d8d1e3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,7 +21,7 @@ use routes::*; mod i18n; use i18n::{i18n_filter, load_catalogs}; -use crate::i18n::langs_filter; +use crate::{i18n::langs_filter, utils::furigana_filter}; mod prelude; @@ -106,6 +106,12 @@ async fn rocket() -> Result, rocket::Error> { move |value: &Value, args: &HashMap| { langs_filter(value, args, &langs) }, + ); + engines.tera.register_filter( + "furigana", + move |value: &Value, args: &HashMap| { + furigana_filter(value, args) + }, ) })) .attach(SassFairing::default()) diff --git a/src/models/challenge.rs b/src/models/challenge.rs index 99a9f39..758b435 100644 --- a/src/models/challenge.rs +++ b/src/models/challenge.rs @@ -1,14 +1,16 @@ use core::panic; use std::str::FromStr; +use comrak::format_html; use serde::{Deserialize, Deserializer, Serialize}; use serde_yaml::Value; -use crate::prelude::*; +use crate::{prelude::*, utils::furigana_to_html}; #[derive(Serialize, Deserialize)] pub struct Challenge { - pub japanese: Vec>>, + pub text: Option, + pub japanese: Option>>>, pub english: Option>, pub song: Option, pub youtube: Option, @@ -29,10 +31,14 @@ impl Challenge { options }; let arena = Arena::new(); + let challenge_text = fs::read_to_string(format!("content/challenges/{number}.md")) + .expect("Couldn't find challenge file"); let root = parse_document( &arena, - &fs::read_to_string(format!("content/challenges/{number}.md")) - .expect("Couldn't find challenge file"), + &(challenge_text + // comrak can't find frontmatter if there's only frontmatter and no newline at end + // TODO: Open issue in comrak + + "\n"), &options, ); if let Some(node) = root.children().next() { @@ -42,7 +48,13 @@ impl Challenge { let lines: Vec<&str> = frontmatter.trim().lines().collect(); lines[1..lines.len() - 1].join("\n") }; - let challenge: Challenge = serde_yaml::from_str(&frontmatter).unwrap(); + let mut challenge: Challenge = serde_yaml::from_str(&frontmatter).unwrap(); + //challenge.text = Some(challenge_text.replace(&frontmatter, "").trim().to_owned()); + let mut html = vec![]; + format_html(root, &ComrakOptions::default(), &mut html) + .expect("Failed to format HTML"); + challenge.text = Some(furigana_to_html(&gh_emoji::Replacer::new() + .replace_all(&String::from_utf8(html).unwrap()))); challenge } else { panic!("No frontmatter!") diff --git a/src/models/database.rs b/src/models/database.rs index 71623e2..49fb316 100644 --- a/src/models/database.rs +++ b/src/models/database.rs @@ -1,7 +1,7 @@ use std::collections::HashSet; use std::{fs::File, io::Read, collections::HashMap, path::Path}; -use poise::serenity_prelude::Http; +use poise::serenity_prelude::{SerenityError, Http}; use r2d2::{Pool, PooledConnection}; use r2d2_sqlite::SqliteConnectionManager; use r2d2_sqlite::rusqlite::{self, params}; @@ -25,6 +25,12 @@ pub enum DatabaseError { Pool(r2d2::Error), } +#[derive(From, Debug)] +pub enum LoadLegacyError { + Database(DatabaseError), + Serenity(SerenityError), +} + type Result = std::result::Result; impl Database { @@ -58,7 +64,8 @@ impl Database { id INTEGER PRIMARY KEY, name TEXT NOT NULL, discriminator INTEGER NOT NULL, - avatar TEXT + avatar TEXT, + deleted INTEGER NOT NULL )", params![], )?; @@ -76,85 +83,80 @@ impl Database { .is_ok()) } - pub async fn load_legacy(&self, http: &Http) -> Result<()> { + pub async fn load_legacy(&self, http: &Http) -> std::result::Result<(), LoadLegacyError> { let latest_challenge = get_challenge_number(); // HashMap of archived users that are no longer sharing a server with 字ちゃん // Their historical usernames and discriminators will be used let mut archived_users = HashMap::new(); - let conn = self.conn()?; + let conn = self.conn().map_err(DatabaseError::Pool)?; for n in 1..=latest_challenge { println!("Loading legacy challenge {n}/{latest_challenge}..."); let mut file = File::open(format!("data/challenges/{n}.json")).unwrap(); let mut contents = String::new(); file.read_to_string(&mut contents).unwrap(); - for (legacy, submissions) in serde_json::from_str::>(&contents) - .unwrap() - .iter() - .map(|legacy| (legacy, legacy.parse().unwrap())) { - let mut already_updated = false; - for submission in submissions { - conn.execute( - "INSERT INTO Submission(author_id, timestamp, image, challenge) VALUES (?1, ?2, ?3, ?4)", - params![ - &submission.author_id, - &submission.timestamp, - &submission.image, - n, - ] - )?; - let id = submission.author_id; - if !self.has_submitted(id)? { - println!("Fetching user {id}..."); - let previously_archived = archived_users.contains_key(&id); - // Parse archived user out of legacy and insert into HashMap - let mut archive = || { - if already_updated { - return; - } - if previously_archived { - println!("Updating archived data for user {id}"); - } else { - println!("Adding archived data for user {id}"); - } - let (name, discriminator) = { - let mut iter = legacy.username.split('#'); - let name = iter.next().unwrap().to_owned(); - let discriminator = iter - .next() - .map(|str| str.parse().unwrap()) - .unwrap_or(0); - (name, discriminator) - }; + for legacy in serde_json::from_str::>(&contents).unwrap() { + let id = legacy.id; + if !self.has_submitted(id)? { + println!("Fetching user {id}..."); + // If it already contains the archived user, + // overwrite write their data since they may have updated + // their username/discriminator since their previous submission + match archived_users.get(&id) { + Some(User { deleted, .. }) => { + archived_users.insert(id, User { + id, + avatar: None, + deleted: *deleted, + ..User::from_username(&legacy.username) + }); + }, + None => match User::fetch(http, id).await { + Ok(User { deleted: true, .. }) => { archived_users.insert(id, User { id, - name, - discriminator, avatar: None, + deleted: true, + ..User::from_username(&legacy.username) }); - already_updated = true; - }; - if previously_archived { - // If it already contains the archived user, - // overwrite write their data since they may have updated - // their username/discriminator since their previous submission - archive(); - } else { - match User::fetch(http, submission.author_id).await { - Ok(user) => { - conn.execute( - "INSERT INTO User(id, name, discriminator, avatar) VALUES (?1, ?2, ?3, ?4)", - params![user.id, user.name, user.discriminator, user.avatar] - )?; - }, - Err(error) => { - println!("Failed to fetch user {}, may update archived data: {error}", submission.author_id); - archive(); - }, - }; - } - } - } + }, + Ok(user) => { + conn.execute( + "INSERT INTO User(id, name, discriminator, avatar, deleted) VALUES (?1, ?2, ?3, ?4, ?5)", + params![user.id, user.name, user.discriminator, user.avatar, user.deleted] + ).map_err(DatabaseError::Rusqlite)?; + }, + Err(error) if error.to_string().eq("Unknown User") => { + // This will also be called in the case of an invalid user ID + println!("Failed to fetch user {id}, adding to archive"); + archived_users.insert(id, User { + id, + avatar: None, + deleted: false, + ..User::from_username(&legacy.username) + }); + }, + Err(error) => return Err(LoadLegacyError::Serenity(error)), + }, + }; } + for submission in legacy.parse().unwrap() { + conn.execute( + "INSERT INTO Submission(author_id, timestamp, image, challenge) VALUES (?1, ?2, ?3, ?4)", + params![ + &submission.author_id, + &submission.timestamp, + &submission.image, + n, + ] + ).map_err(DatabaseError::Rusqlite)?; + } + } + } + for (_id, user) in archived_users { + conn.execute( + "INSERT INTO USER (id, name, discriminator, avatar, deleted) VALUES (?1, ?2, ?3, ?4, ?5)", + params![user.id, user.name, user.discriminator, user.avatar, user.deleted] + ).map_err(DatabaseError::Rusqlite)?; } Ok(()) } @@ -191,13 +193,14 @@ impl Database { .iter() // u64 must be converted to String for templates .map(|id| -> Result<(String, User)> { - match conn.prepare("SELECT name, discriminator, avatar FROM User WHERE id = ?1") { + match conn.prepare("SELECT name, discriminator, avatar, deleted FROM User WHERE id = ?1") { Ok(mut statement) => statement.query_row(params![id], |row| { Ok((id.to_string(), User { id: *id, name: row.get(0)?, discriminator: row.get(1)?, avatar: row.get(2)?, + deleted: row.get(3)?, })) }).map_err(DatabaseError::Rusqlite), Err(error) => Err(DatabaseError::Rusqlite(error)), diff --git a/src/models/submission/legacy_submission.rs b/src/models/submission/legacy_submission.rs index fcf4008..17b9743 100644 --- a/src/models/submission/legacy_submission.rs +++ b/src/models/submission/legacy_submission.rs @@ -6,11 +6,20 @@ use super::Submission; #[derive(Deserialize)] pub struct LegacySubmission { - pub id: String, + #[serde(deserialize_with = "deserialize_id")] + pub id: u64, pub images: Vec, pub username: String, } +pub fn deserialize_id<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let id_str: &str = serde::Deserialize::deserialize(deserializer)?; + id_str.parse().map_err(serde::de::Error::custom) +} + #[derive(From, Debug)] pub enum LegacySubmissionParseError { BadAuthorId(std::num::ParseIntError), @@ -18,7 +27,7 @@ pub enum LegacySubmissionParseError { impl LegacySubmission { pub fn parse(&self) -> Result, LegacySubmissionParseError> { - let author_id = self.id.parse()?; + let author_id = self.id; Ok(self.images .iter() .map(|image| { diff --git a/src/models/user/mod.rs b/src/models/user/mod.rs index 073449c..c9da86d 100644 --- a/src/models/user/mod.rs +++ b/src/models/user/mod.rs @@ -6,6 +6,7 @@ mod tests; use chrono::Utc; use derive_more::From; use poise::serenity_prelude::{self, UserId, Http}; +use regex::Regex; use reqwest::StatusCode; use rocket::http::{Cookie, CookieJar}; use serial::*; @@ -27,6 +28,7 @@ pub struct User { #[serde(deserialize_with = "deserialize_discriminator")] pub discriminator: u16, pub avatar: Option, + pub deleted: bool, } impl Username for User { @@ -38,12 +40,36 @@ impl Username for User { } } +fn is_name_deleted(name: &str) -> bool { + Regex::new(r"Deleted User [a-f0-9]{8}").unwrap().is_match(name) +} + impl User { + pub fn from_username(username: &str) -> Self { + let (name, discriminator) = { + let mut iter = username.split('#'); + let name = iter.next().unwrap().to_owned(); + let discriminator = iter + .next() + .map(|str| str.parse().unwrap()) + .unwrap_or(0); + (name, discriminator) + }; + Self { + id: 0, + name, + discriminator, + avatar: None, + deleted: false, + } + } + pub async fn fetch(http: &Http, id: u64) -> serenity_prelude::Result { let user = UserId(id).to_user(http).await?; Ok(Self { id, avatar: user.avatar, + deleted: is_name_deleted(&user.name), name: user.name, discriminator: user.discriminator, }) @@ -51,7 +77,6 @@ impl User { fn avatar(&self) -> String { match &self.avatar { - // https://cdn.discordapp.com/avatars/67795786229878784/e58524afe21b5058fc6f3cdc19aea8e1.webp?size=1024 Some(avatar) => format!( "https://cdn.discordapp.com/avatars/{}/{}.{}?size=1024", self.id, @@ -118,6 +143,7 @@ impl SessionUser { name: parse_cookie_value(cookies, USER_NAME_COOKIE)?, discriminator: parse_cookie_value(cookies, USER_DISCRIMINATOR_COOKIE)?, avatar: Some(parse_cookie_value(cookies, USER_AVATAR_COOKIE)?), + deleted: false, })) } diff --git a/src/models/user/serial.rs b/src/models/user/serial.rs index c330e4d..f738053 100644 --- a/src/models/user/serial.rs +++ b/src/models/user/serial.rs @@ -23,11 +23,12 @@ impl Serialize for User { where S: Serializer, { - let mut state = serializer.serialize_struct("User", 5)?; + let mut state = serializer.serialize_struct("User", 6)?; state.serialize_field("id", &self.id)?; state.serialize_field("name", &self.name)?; state.serialize_field("discriminator", &self.discriminator)?; state.serialize_field("avatar", &self.avatar())?; + state.serialize_field("deleted", &self.deleted)?; state.serialize_field("username", &self.username())?; state.end() } diff --git a/src/models/user/tests.rs b/src/models/user/tests.rs index b60b5f7..8376850 100644 --- a/src/models/user/tests.rs +++ b/src/models/user/tests.rs @@ -19,3 +19,12 @@ fn test_new_username() { let user = test_user("test", 0); assert_eq!(user.username(), "test"); } + +#[test] +fn is_name_deleted() { + use super::is_name_deleted; + assert!(is_name_deleted("Deleted User ce34a7da")); + assert!(!is_name_deleted("Deleted User Ce34a7da")); // capital letter in hex + assert!(!is_name_deleted("Deleted User ce34a7d")); // hex too short + assert!(!is_name_deleted("Deleted User")); // no hex +} \ No newline at end of file diff --git a/src/utils/furigana.rs b/src/utils/furigana.rs new file mode 100644 index 0000000..749ca08 --- /dev/null +++ b/src/utils/furigana.rs @@ -0,0 +1,39 @@ +use std::collections::HashMap; + +use regex::Regex; +use rocket_dyn_templates::tera::{self, Value}; + +pub fn furigana_to_html(text: &str) -> String { + // Original regular expression: \[([^\]]*)\]{([^\}]*)} + // https://regexr.com/6dspa + // https://blog.elnu.com/2022/01/furigana-in-markdown-using-regular-expressions/ + // The regex crate users curly braces {} as repetition qualifiers, + // so { needs to be escaped as \{ + // Curly brace literals \{ need to have their backslash escaped as \\{ + // TODO: Modify so only wraps continuous sections of furigana + let re = Regex::new(r"\[([^\]]*)\]\{([^\\}]*)}").unwrap(); + format!("{}", re.replace_all(text, "$1($2)")) +} + +pub fn furigana_filter(value: &Value, _args: &HashMap) -> tera::Result { + if value.is_null() { + return Ok(Value::String("".to_string())); + } + Ok(Value::String(furigana_to_html(value.as_str().expect("The furigana input must be a string")))) +} + +#[cfg(test)] +mod tests { + #[test] + fn furigana_to_html() { + use super::furigana_to_html; + assert_eq!( + furigana_to_html("[振]{ふ}り[仮]{が}[名]{な}"), + "\ + 振(ふ)り\ + 仮(が)\ + 名(な)\ + " + ); + } +} \ No newline at end of file diff --git a/src/utils/mod.rs b/src/utils/mod.rs index f8b78ae..5702a3b 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -5,4 +5,7 @@ mod headers; pub use headers::*; mod challenge; -pub use challenge::*; \ No newline at end of file +pub use challenge::*; + +mod furigana; +pub use furigana::*; \ No newline at end of file diff --git a/templates/index.html.tera b/templates/index.html.tera index 790e832..7272dd5 100644 --- a/templates/index.html.tera +++ b/templates/index.html.tera @@ -19,7 +19,7 @@ {% if content.song %}
- {{ content.song.japanese }} + {% if content.song.japanese %}{{ content.song.japanese | furigana | safe }}{% else %}{{ content.song.english }}{% endif %}
{% endif %} {% if user %} @@ -67,6 +67,7 @@

Welcome to Tegaki Tuesday #{{ challenge }}!

+ {% if content.japanese %}