cargo clippy --fix

Cargo.lock

@@ -1120,6 +1120,16 @@ dependencies = [
  "encoding",
 ]
 
+[[package]]
+name = "gh-emoji"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30ad64b43d48c1745c0059e5ba223816eb599eec8956c668fc0a31f6b9cd799e"
+dependencies = [
+ "phf",
+ "regex",
+]
+
 [[package]]
 name = "ghash"
 version = "0.5.0"
@@ -2502,9 +2512,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.8.3"
+version = "1.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390"
+checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
 dependencies = [
  "aho-corasick 1.0.1",
  "memchr",
@@ -3276,9 +3286,11 @@ dependencies = [
  "derive_more",
  "dotenv",
  "gettext",
+ "gh-emoji",
  "poise",
  "r2d2",
  "r2d2_sqlite",
+ "regex",
  "reqwest",
  "rocket 0.5.0-rc.3",
  "rocket_contrib",

Cargo.toml

@@ -13,9 +13,11 @@ comrak = "0.18.0"
 derive_more = "0.99.17"
 dotenv = "0.15.0"
 gettext = "0.4.0"
+gh-emoji = "1.0.7"
 poise = "0.5.5"
 r2d2 = "0.8.10"
 r2d2_sqlite = "0.22.0"
+regex = "1.8.4"
 reqwest = "0.11.18"
 rocket = { version = "=0.5.0-rc.3", features = ["secrets", "json"] }
 rocket_contrib = { version = "0.4.11", features = ["templates"] }

content/challenges/110.md

@@ -17,7 +17,6 @@ japanese:
           - 霊: れい
       - pos: particle
         text: は
-      - pos: particle
       - pos: verb
         text:
           - 踊: おど

content/challenges/66.md

@@ -1,8 +1,8 @@
 ---
 japanese:
-  - - - pos: noun
+  - - - pos: verb
         text: スイーツ
-        pos: verb
+      - pos: verb
         text:
           - 食: た
           - べ
@@ -89,3 +89,5 @@ youtube: IqZ2qcE3XVc
 suggester: 猛火妹紅（MochaMoko）
 date: 2022-06-17
 ---
+
+**Note 2023-07-01:** The first word of the first line, スイーツ, was missing in the original challenge and has been corrected.

content/challenges/99.md

@@ -101,4 +101,4 @@ suggester: モカ妹紅（MochaMoko）
 date: 2023-02-14
 ---
 
-{{< banner >}}:love_letter: Happy Valentine's Day!{{< /banner >}}
+:love_letter: Happy Valentine's Day!

src/main.rs

@@ -21,7 +21,7 @@ use routes::*;
 mod i18n;
 use i18n::{i18n_filter, load_catalogs};
 
-use crate::i18n::langs_filter;
+use crate::{i18n::langs_filter, utils::furigana_filter};
 
 mod prelude;
 
@@ -106,6 +106,12 @@ async fn rocket() -> Result<Rocket<Ignite>, rocket::Error> {
                 move |value: &Value, args: &HashMap<String, Value>| {
                     langs_filter(value, args, &langs)
                 },
+            );
+            engines.tera.register_filter(
+                "furigana",
+                move |value: &Value, args: &HashMap<String, Value>| {
+                    furigana_filter(value, args)
+                },
             )
         }))
         .attach(SassFairing::default())

src/models/challenge.rs

@@ -1,14 +1,16 @@
 use core::panic;
 use std::str::FromStr;
 
+use comrak::format_html;
 use serde::{Deserialize, Deserializer, Serialize};
 use serde_yaml::Value;
 
-use crate::prelude::*;
+use crate::{prelude::*, utils::furigana_to_html};
 
 #[derive(Serialize, Deserialize)]
 pub struct Challenge {
-    pub japanese: Vec<Vec<Vec<ChallengeWord>>>,
+    pub text: Option<String>,
+    pub japanese: Option<Vec<Vec<Vec<ChallengeWord>>>>,
     pub english: Option<Vec<String>>,
     pub song: Option<Song>,
     pub youtube: Option<String>,
@@ -29,10 +31,14 @@ impl Challenge {
             options
         };
         let arena = Arena::new();
+        let challenge_text = fs::read_to_string(format!("content/challenges/{number}.md"))
+            .expect("Couldn't find challenge file");
         let root = parse_document(
             &arena,
-            &fs::read_to_string(format!("content/challenges/{number}.md"))
-                .expect("Couldn't find challenge file"),
+           &(challenge_text
+                // comrak can't find frontmatter if there's only frontmatter and no newline at end
+                // TODO: Open issue in comrak
+                + "\n"),
             &options,
         );
         if let Some(node) = root.children().next() {
@@ -42,7 +48,13 @@ impl Challenge {
                     let lines: Vec<&str> = frontmatter.trim().lines().collect();
                     lines[1..lines.len() - 1].join("\n")
                 };
-                let challenge: Challenge = serde_yaml::from_str(&frontmatter).unwrap();
+                let mut challenge: Challenge = serde_yaml::from_str(&frontmatter).unwrap();
+                //challenge.text = Some(challenge_text.replace(&frontmatter, "").trim().to_owned());
+                let mut html = vec![];
+                format_html(root, &ComrakOptions::default(), &mut html)
+                    .expect("Failed to format HTML");
+                challenge.text = Some(furigana_to_html(&gh_emoji::Replacer::new()
+                    .replace_all(&String::from_utf8(html).unwrap())));
                 challenge
             } else {
                 panic!("No frontmatter!")

src/models/database.rs

@@ -1,7 +1,7 @@
 use std::collections::HashSet;
 use std::{fs::File, io::Read, collections::HashMap, path::Path};
 
-use poise::serenity_prelude::Http;
+use poise::serenity_prelude::{SerenityError, Http};
 use r2d2::{Pool, PooledConnection};
 use r2d2_sqlite::SqliteConnectionManager;
 use r2d2_sqlite::rusqlite::{self, params};
@@ -25,6 +25,12 @@ pub enum DatabaseError {
     Pool(r2d2::Error),
 }
 
+#[derive(From, Debug)]
+pub enum LoadLegacyError {
+    Database(DatabaseError),
+    Serenity(SerenityError),
+}
+
 type Result<T> = std::result::Result<T, DatabaseError>;
 
 impl Database {
@@ -58,7 +64,8 @@ impl Database {
                 id            INTEGER PRIMARY KEY,
                 name          TEXT NOT NULL,
                 discriminator INTEGER NOT NULL,
-                avatar        TEXT
+                avatar        TEXT,
+                deleted       INTEGER NOT NULL
             )",
             params![],
         )?;
@@ -76,85 +83,80 @@ impl Database {
             .is_ok())
     }
 
-    pub async fn load_legacy(&self, http: &Http) -> Result<()> {
+    pub async fn load_legacy(&self, http: &Http) -> std::result::Result<(), LoadLegacyError> {
         let latest_challenge = get_challenge_number();
         // HashMap of archived users that are no longer sharing a server with 字ちゃん
         // Their historical usernames and discriminators will be used
         let mut archived_users = HashMap::new();
-        let conn = self.conn()?;
+        let conn = self.conn().map_err(DatabaseError::Pool)?;
         for n in 1..=latest_challenge {
             println!("Loading legacy challenge {n}/{latest_challenge}...");
             let mut file = File::open(format!("data/challenges/{n}.json")).unwrap();
             let mut contents = String::new();
             file.read_to_string(&mut contents).unwrap();
-            for (legacy, submissions) in serde_json::from_str::<Vec<LegacySubmission>>(&contents)
-                .unwrap()
-                .iter()
-                .map(|legacy| (legacy, legacy.parse().unwrap())) {
-                    let mut already_updated = false;
-                    for submission in submissions {
-                        conn.execute(
-                            "INSERT INTO Submission(author_id, timestamp, image, challenge) VALUES (?1, ?2, ?3, ?4)",
-                            params![
-                                &submission.author_id,
-                                &submission.timestamp,
-                                &submission.image,
-                                n,
-                            ]
-                        )?;
-                        let id = submission.author_id;
-                        if !self.has_submitted(id)? {
-                            println!("Fetching user {id}...");
-                            let previously_archived = archived_users.contains_key(&id);
-                            // Parse archived user out of legacy and insert into HashMap
-                            let mut archive = || {
-                                if already_updated {
-                                    return;
-                                }
-                                if previously_archived {
-                                    println!("Updating archived data for user {id}");
-                                } else {
-                                    println!("Adding archived data for user {id}");
-                                }
-                                let (name, discriminator) = {
-                                    let mut iter = legacy.username.split('#');
-                                    let name = iter.next().unwrap().to_owned();
-                                    let discriminator = iter
-                                        .next()
-                                        .map(|str| str.parse().unwrap())
-                                        .unwrap_or(0);
-                                    (name, discriminator)
-                                };
+            for legacy in serde_json::from_str::<Vec<LegacySubmission>>(&contents).unwrap() {
+                let id = legacy.id;
+                if !self.has_submitted(id)? {
+                    println!("Fetching user {id}...");
+                    // If it already contains the archived user,
+                    // overwrite write their data since they may have updated
+                    // their username/discriminator since their previous submission
+                    match archived_users.get(&id) {
+                        Some(User { deleted, .. }) => {
+                            archived_users.insert(id, User {
+                                id,
+                                avatar: None,
+                                deleted: *deleted,
+                                ..User::from_username(&legacy.username)
+                            });
+                        },
+                        None => match User::fetch(http, id).await {
+                            Ok(User { deleted: true, .. }) => {
                                 archived_users.insert(id, User {
                                     id,
-                                    name,
-                                    discriminator,
                                     avatar: None,
+                                    deleted: true,
+                                    ..User::from_username(&legacy.username)
                                 });
-                                already_updated = true;
-                            };
-                            if previously_archived {
-                                // If it already contains the archived user,
-                                // overwrite write their data since they may have updated
-                                // their username/discriminator since their previous submission
-                                archive();
-                            } else {
-                                match User::fetch(http, submission.author_id).await {
-                                    Ok(user) => {
-                                        conn.execute(
-                                            "INSERT INTO User(id, name, discriminator, avatar) VALUES (?1, ?2, ?3, ?4)",
-                                            params![user.id, user.name, user.discriminator, user.avatar]
-                                        )?;
-                                    },
-                                    Err(error) => {
-                                        println!("Failed to fetch user {}, may update archived data: {error}", submission.author_id);
-                                        archive();
-                                    },
-                                };
-                            }
-                        }
-                    }
+                            },
+                            Ok(user) => {
+                                conn.execute(
+                                    "INSERT INTO User(id, name, discriminator, avatar, deleted) VALUES (?1, ?2, ?3, ?4, ?5)",
+                                    params![user.id, user.name, user.discriminator, user.avatar, user.deleted]
+                                ).map_err(DatabaseError::Rusqlite)?;
+                            },
+                            Err(error) if error.to_string().eq("Unknown User") => {
+                                // This will also be called in the case of an invalid user ID
+                                println!("Failed to fetch user {id}, adding to archive");
+                                archived_users.insert(id, User {
+                                    id,
+                                    avatar: None,
+                                    deleted: false,
+                                    ..User::from_username(&legacy.username)
+                                });
+                            },
+                            Err(error) => return Err(LoadLegacyError::Serenity(error)),
+                        },
+                    };
                 }
+                for submission in legacy.parse().unwrap() {
+                    conn.execute(
+                        "INSERT INTO Submission(author_id, timestamp, image, challenge) VALUES (?1, ?2, ?3, ?4)",
+                        params![
+                            &submission.author_id,
+                            &submission.timestamp,
+                            &submission.image,
+                            n,
+                        ]
+                    ).map_err(DatabaseError::Rusqlite)?;
+                }
+            }
+        }
+        for (_id, user) in archived_users {
+            conn.execute(
+                "INSERT INTO USER (id, name, discriminator, avatar, deleted) VALUES (?1, ?2, ?3, ?4, ?5)",
+                params![user.id, user.name, user.discriminator, user.avatar, user.deleted]
+            ).map_err(DatabaseError::Rusqlite)?;
         }
         Ok(())
     }
@@ -191,13 +193,14 @@ impl Database {
             .iter()
             // u64 must be converted to String for templates
             .map(|id| -> Result<(String, User)> {
-                match conn.prepare("SELECT name, discriminator, avatar FROM User WHERE id = ?1") {
+                match conn.prepare("SELECT name, discriminator, avatar, deleted FROM User WHERE id = ?1") {
                     Ok(mut statement) => statement.query_row(params![id], |row| {
                         Ok((id.to_string(), User {
                             id: *id,
                             name: row.get(0)?,
                             discriminator: row.get(1)?,
                             avatar: row.get(2)?,
+                            deleted: row.get(3)?,
                         }))
                     }).map_err(DatabaseError::Rusqlite),
                     Err(error) => Err(DatabaseError::Rusqlite(error)),

src/models/submission/legacy_submission.rs

@@ -6,11 +6,20 @@ use super::Submission;
 
 #[derive(Deserialize)]
 pub struct LegacySubmission {
-    pub id: String,
+    #[serde(deserialize_with = "deserialize_id")]
+    pub id: u64,
     pub images: Vec<String>,
     pub username: String,
 }
 
+pub fn deserialize_id<'de, D>(deserializer: D) -> Result<u64, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let id_str: &str = serde::Deserialize::deserialize(deserializer)?;
+    id_str.parse().map_err(serde::de::Error::custom)
+}
+
 #[derive(From, Debug)]
 pub enum LegacySubmissionParseError {
     BadAuthorId(std::num::ParseIntError),
@@ -18,7 +27,7 @@ pub enum LegacySubmissionParseError {
 
 impl LegacySubmission {
     pub fn parse(&self) -> Result<Vec<Submission>, LegacySubmissionParseError> {
-        let author_id = self.id.parse()?;
+        let author_id = self.id;
         Ok(self.images
             .iter()
             .map(|image| {

src/models/user/mod.rs

@@ -6,6 +6,7 @@ mod tests;
 use chrono::Utc;
 use derive_more::From;
 use poise::serenity_prelude::{self, UserId, Http};
+use regex::Regex;
 use reqwest::StatusCode;
 use rocket::http::{Cookie, CookieJar};
 use serial::*;
@@ -27,6 +28,7 @@ pub struct User {
     #[serde(deserialize_with = "deserialize_discriminator")]
     pub discriminator: u16,
     pub avatar: Option<String>,
+    pub deleted: bool,
 }
 
 impl Username for User {
@@ -38,12 +40,36 @@ impl Username for User {
     }
 }
 
+fn is_name_deleted(name: &str) -> bool {
+    Regex::new(r"Deleted User [a-f0-9]{8}").unwrap().is_match(name)
+}
+
 impl User {
+    pub fn from_username(username: &str) -> Self {
+        let (name, discriminator) = {
+            let mut iter = username.split('#');
+            let name = iter.next().unwrap().to_owned();
+            let discriminator = iter
+                .next()
+                .map(|str| str.parse().unwrap())
+                .unwrap_or(0);
+            (name, discriminator)
+        };
+        Self {
+            id: 0,
+            name,
+            discriminator,
+            avatar: None,
+            deleted: false,
+        }
+    }
+
     pub async fn fetch(http: &Http, id: u64) -> serenity_prelude::Result<Self> {
         let user = UserId(id).to_user(http).await?;
         Ok(Self {
             id,
             avatar: user.avatar,
+            deleted: is_name_deleted(&user.name),
             name: user.name,
             discriminator: user.discriminator,
         })
@@ -51,7 +77,6 @@ impl User {
 
     fn avatar(&self) -> String {
         match &self.avatar {
-            // https://cdn.discordapp.com/avatars/67795786229878784/e58524afe21b5058fc6f3cdc19aea8e1.webp?size=1024
             Some(avatar) => format!(
                 "https://cdn.discordapp.com/avatars/{}/{}.{}?size=1024",
                 self.id,
@@ -118,6 +143,7 @@ impl SessionUser {
             name: parse_cookie_value(cookies, USER_NAME_COOKIE)?,
             discriminator: parse_cookie_value(cookies, USER_DISCRIMINATOR_COOKIE)?,
             avatar: Some(parse_cookie_value(cookies, USER_AVATAR_COOKIE)?),
+            deleted: false,
         }))
     }
 

src/models/user/serial.rs

@@ -23,11 +23,12 @@ impl Serialize for User {
     where
         S: Serializer,
     {
-        let mut state = serializer.serialize_struct("User", 5)?;
+        let mut state = serializer.serialize_struct("User", 6)?;
         state.serialize_field("id", &self.id)?;
         state.serialize_field("name", &self.name)?;
         state.serialize_field("discriminator", &self.discriminator)?;
         state.serialize_field("avatar", &self.avatar())?;
+        state.serialize_field("deleted", &self.deleted)?;
         state.serialize_field("username", &self.username())?;
         state.end()
     }

src/models/user/tests.rs

@@ -19,3 +19,12 @@ fn test_new_username() {
     let user = test_user("test", 0);
     assert_eq!(user.username(), "test");
 }
+
+#[test]
+fn is_name_deleted() {
+    use super::is_name_deleted;
+    assert!(is_name_deleted("Deleted User ce34a7da"));
+    assert!(!is_name_deleted("Deleted User Ce34a7da")); // capital letter in hex
+    assert!(!is_name_deleted("Deleted User ce34a7d")); // hex too short
+    assert!(!is_name_deleted("Deleted User")); // no hex
+}

src/utils/furigana.rs

@@ -0,0 +1,39 @@
+use std::collections::HashMap;
+
+use regex::Regex;
+use rocket_dyn_templates::tera::{self, Value};
+
+pub fn furigana_to_html(text: &str) -> String {
+    // Original regular expression: \[([^\]]*)\]{([^\}]*)}
+    // https://regexr.com/6dspa
+    // https://blog.elnu.com/2022/01/furigana-in-markdown-using-regular-expressions/
+    // The regex crate users curly braces {} as repetition qualifiers,
+    // so { needs to be escaped as \{
+    // Curly brace literals \{ need to have their backslash escaped as \\{
+    // TODO: Modify so <span lang="ja"> only wraps continuous sections of furigana
+    let re = Regex::new(r"\[([^\]]*)\]\{([^\\}]*)}").unwrap();
+    format!("<span lang=\"ja\">{}</span>", re.replace_all(text, "<ruby>$1<rp>(</rp><rt>$2</rt><rp>)</rp></ruby>"))
+}
+
+pub fn furigana_filter(value: &Value, _args: &HashMap<String, Value>) -> tera::Result<Value> {
+    if value.is_null() {
+        return Ok(Value::String("".to_string()));
+    }
+    Ok(Value::String(furigana_to_html(value.as_str().expect("The furigana input must be a string"))))
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn furigana_to_html() {
+        use super::furigana_to_html;
+        assert_eq!(
+            furigana_to_html("[振]{ふ}り[仮]{が}[名]{な}"),
+            "<span lang=\"ja\">\
+                <ruby>振<rp>(</rp><rt>ふ</rt><rp>)</rp></ruby>り\
+                <ruby>仮<rp>(</rp><rt>が</rt><rp>)</rp></ruby>\
+                <ruby>名<rp>(</rp><rt>な</rt><rp>)</rp></ruby>\
+                </span>"
+        );
+    }
+}

src/utils/mod.rs

@@ -5,4 +5,7 @@ mod headers;
 pub use headers::*;
 
 mod challenge;
-pub use challenge::*;
+pub use challenge::*;
+
+mod furigana;
+pub use furigana::*;

templates/index.html.tera

@@ -19,7 +19,7 @@
             <a class="link" href="/{{ challenge + 1 }}"><svg class="svg-inline" aria-hidden="true" focusable="false" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M342.6 233.4c12.5 12.5 12.5 32.8 0 45.3l-192 192c-12.5 12.5-32.8 12.5-45.3 0s-12.5-32.8 0-45.3L274.7 256 105.4 86.6c-12.5-12.5-12.5-32.8 0-45.3s32.8-12.5 45.3 0l192 192z"></path></svg></a>
             {% if content.song %}
             <div>
-                <span>{{ content.song.japanese }}</span>
+                <span>{% if content.song.japanese %}{{ content.song.japanese | furigana | safe }}{% else %}{{ content.song.english }}{% endif %}</span>
             </div>
             {% endif %}
             {% if user %}
@@ -67,6 +67,7 @@
         <div>
             <div>
                 <h1>Welcome to Tegaki Tuesday #{{ challenge }}!</h1>
+                {% if content.japanese %}
                 <div lang="ja">
                     <script>
                         let kyujitai = false;
@@ -98,6 +99,8 @@
                         {% endfor %}
                     {% endfor %}
                 </div>
+                {% endif %}
+                {{ content.text | safe }}
                 {% if content.translation %}
                     <p>
                         Translation
@@ -139,7 +142,7 @@
                         {% set author = users[submission.author_id] %}
                         <figure>
                             <img src="/{{ challenge }}/{{ submission.image }}" alt="{{ author.username }}'s submission" onclick="submissionModal('{{ submission.image }}')">
-                            <figcaption><a href="https://discord.com/users/{{ author.id }}" target="_blank">{{ author.username }}</a></figcaption>
+                            <figcaption>{% if not author.deleted %}<a href="https://discord.com/users/{{ author.id }}" target="_blank">{% endif %}{{ author.username }}{% if author.deleted %} (deleted account){% else %}</a>{% endif %}</figcaption>
                         </figure>
                     {% endfor %}
                 </div>