diff --git a/Cargo.toml b/Cargo.toml index d1ea206..7e7a9f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,9 @@ ammonia = "3.3" toml = "0.8" syntect = "5.1" deunicode = "1.4" -reqwest = { version = "0.11", features = ["json"] } +reqwest = { version = "0.11", features = ["json", "blocking"] } +scraper = "0.26" +ego-tree = "0.11" serde_json = "1.0" url = "2.4" diff --git a/content/markup.md b/content/markup.md index 050f7f9..43141c1 100644 --- a/content/markup.md +++ b/content/markup.md @@ -73,6 +73,69 @@ You can^Wrap text with carets^ to make it ^superscript!^ > Creates an indented quote block +## Alerts +```md +> [!NOTE] +> Useful information that users should know, even when skimming content. +``` + +> [!NOTE] +> Useful information that users should know, even when skimming content. + +```md +> [!TIP] +> Helpful advice for doing things better or more easily. +``` + +> [!TIP] +> Helpful advice for doing things better or more easily. + +```md +> [!IMPORTANT] +> Key information users need to know to achieve their goal. +``` + +> [!IMPORTANT] +> Key information users need to know to achieve their goal. + +```md +> [!WARNING] +> Urgent info that needs immediate user attention to avoid problems. +``` + +> [!WARNING] +> Urgent info that needs immediate user attention to avoid problems. + +```md +> [!CAUTION] +> Advises about risks or negative outcomes of certain actions. +``` + +> [!CAUTION] +> Advises about risks or negative outcomes of certain actions. + +### Custom Color Alerts + +Use a 3- or 6-digit hex color and a custom label (up to 25 characters) separated by `!`: + +```md +> [F80!Heads Up] +> A custom orange alert with a short hex color. +``` + +> [F80!Heads Up] +> A custom orange alert with a short hex color. + +```md +> [e600e6!Pro Tip] +> A custom purple alert with a full hex color. +``` + +> [e600e6!Pro Tip] +> A custom purple alert with a full hex color. + +The label text is sanitized to alphanumeric characters, spaces, hyphens, and underscores. + ## Footnotes **Reference footnotes** - mark position with `[^1]` and define at bottom: diff --git a/src/archiver.rs b/src/archiver.rs index 401a7ee..6d52b0f 100644 --- a/src/archiver.rs +++ b/src/archiver.rs @@ -1,8 +1,12 @@ use chrono::{DateTime, Utc}; +use ego_tree::NodeRef; +use scraper::{Html, Node as ScraperNode, Selector}; use serde::Deserialize; use std::collections::HashMap; use std::fs; +// ─── Telegraph types ────────────────────────────────────────────────────────── + #[derive(Debug, Deserialize)] struct TelegraphResponse { ok: bool, @@ -39,6 +43,18 @@ struct NodeElement { children: Option>, } +// ─── Generic article type (used for arbitrary URLs) ─────────────────────────── + +struct GenericPage { + url: String, + title: String, + author: Option, + date: Option, + content_html: String, +} + +// ─── Archiver ───────────────────────────────────────────────────────────────── + pub struct TelegraphArchiver; impl TelegraphArchiver { @@ -46,42 +62,43 @@ impl TelegraphArchiver { Self } + /// Archive any URL. Telegraph pages use the structured API; everything + /// else is scraped as HTML and converted to Nonograph markdown. pub async fn archive_url(&self, url: &str) -> Result> { - // Extract path from Telegraph URL - let path = self.extract_path_from_url(url)?; + let is_telegraph = { + let parsed = url::Url::parse(url)?; + parsed.host_str() == Some("telegra.ph") + }; - // Fetch content from Telegraph API - let page = self.fetch_telegraph_page(&path).await?; + if is_telegraph { + self.archive_telegraph_url(url).await + } else { + self.archive_generic_url(url).await + } + } - // Convert to Nonograph markdown - let markdown = self.convert_to_markdown(&page)?; + // ── Telegraph path ──────────────────────────────────────────────────────── - // Generate filename and save + async fn archive_telegraph_url(&self, url: &str) -> Result> { + let path = self.extract_path_from_url(url)?; + let page = self.fetch_telegraph_page(&path).await?; + let markdown = self.convert_to_markdown(&page)?; let filename = self.generate_filename(&page); let file_path = format!("content/{}", filename); - - // Save to content directory fs::write(&file_path, markdown)?; - - // Return Nonograph URL let nonograph_id = filename.trim_end_matches(".md"); Ok(format!("/{}", nonograph_id)) } fn extract_path_from_url(&self, url: &str) -> Result> { let parsed_url = url::Url::parse(url)?; - - // Verify it's a Telegraph URL if parsed_url.host_str() != Some("telegra.ph") { return Err("URL is not a Telegraph page".into()); } - - // Extract path (remove leading slash) let path = parsed_url.path().trim_start_matches('/'); if path.is_empty() { return Err("Invalid Telegraph URL - no path found".into()); } - Ok(path.to_string()) } @@ -93,17 +110,14 @@ impl TelegraphArchiver { "https://api.telegra.ph/getPage/{}?return_content=true", path ); - let response = reqwest::get(&api_url).await?; let telegraph_response: TelegraphResponse = response.json().await?; - if !telegraph_response.ok { return Err(format!("Telegraph API error: {:?}", telegraph_response.error).into()); } - telegraph_response .result - .ok_or("No result in Telegraph response".into()) + .ok_or_else(|| "No result in Telegraph response".into()) } fn convert_to_markdown( @@ -111,31 +125,27 @@ impl TelegraphArchiver { page: &TelegraphPage, ) -> Result> { let mut markdown = String::new(); - - // Use current date as archival date let now: DateTime = Utc::now(); - let archival_date = format!("{}", now.format("%B %d, %Y")); + let archival_date = now.format("%B %d, %Y").to_string(); - // Add date and author in proper format if let Some(author) = &page.author_name { markdown.push_str(&format!("{} | {}\n\n", archival_date, author)); } else { markdown.push_str(&format!("{}\n\n", archival_date)); } - // Add title as H1 markdown.push_str(&format!("# {}\n", page.title)); - // Convert content + // Source comment (hidden from HTML output) + markdown.push_str(&format!("// Source: {}\n\n", page.url)); + if let Some(content) = &page.content { for node in content { self.convert_node_to_markdown(node, &mut markdown, 0)?; } } - // Clean up excessive newlines - let cleaned = self.clean_excessive_newlines(&markdown); - Ok(cleaned) + Ok(self.clean_excessive_newlines(&markdown)) } fn convert_node_to_markdown( @@ -174,7 +184,7 @@ impl TelegraphArchiver { output.push_str("\n\n"); } "br" => { - output.push_str("\n"); + output.push('\n'); } "strong" | "b" => { output.push_str("**"); @@ -191,7 +201,7 @@ impl TelegraphArchiver { output.push_str("**"); } "em" | "i" => { - output.push_str("*"); + output.push('*'); if let Some(children) = &element.children { for child in children { self.convert_node_to_markdown_with_context( @@ -202,10 +212,10 @@ impl TelegraphArchiver { )?; } } - output.push_str("*"); + output.push('*'); } "u" => { - output.push_str("_"); + output.push('_'); if let Some(children) = &element.children { for child in children { self.convert_node_to_markdown_with_context( @@ -216,10 +226,10 @@ impl TelegraphArchiver { )?; } } - output.push_str("_"); + output.push('_'); } "s" => { - output.push_str("~"); + output.push('~'); if let Some(children) = &element.children { for child in children { self.convert_node_to_markdown_with_context( @@ -230,10 +240,10 @@ impl TelegraphArchiver { )?; } } - output.push_str("~"); + output.push('~'); } "code" => { - output.push_str("`"); + output.push('`'); if let Some(children) = &element.children { for child in children { self.convert_node_to_markdown_with_context( @@ -244,7 +254,7 @@ impl TelegraphArchiver { )?; } } - output.push_str("`"); + output.push('`'); } "pre" => { output.push_str("```\n"); @@ -263,7 +273,7 @@ impl TelegraphArchiver { "a" => { if let Some(attrs) = &element.attrs { if let Some(href) = attrs.get("href") { - output.push_str("["); + output.push('['); if let Some(children) = &element.children { for child in children { self.convert_node_to_markdown_with_context( @@ -293,6 +303,34 @@ impl TelegraphArchiver { } } } + "h1" => { + output.push_str("# "); + if let Some(children) = &element.children { + for child in children { + self.convert_node_to_markdown_with_context( + child, + output, + depth, + image_caption, + )?; + } + } + output.push_str("\n\n"); + } + "h2" => { + output.push_str("## "); + if let Some(children) = &element.children { + for child in children { + self.convert_node_to_markdown_with_context( + child, + output, + depth, + image_caption, + )?; + } + } + output.push_str("\n\n"); + } "h3" => { output.push_str("### "); if let Some(children) = &element.children { @@ -307,7 +345,7 @@ impl TelegraphArchiver { } output.push_str("\n\n"); } - "h4" => { + "h4" | "h5" | "h6" => { output.push_str("#### "); if let Some(children) = &element.children { for child in children { @@ -346,7 +384,7 @@ impl TelegraphArchiver { )?; } } - output.push_str("\n"); + output.push('\n'); } "ol" => { if let Some(children) = &element.children { @@ -364,15 +402,14 @@ impl TelegraphArchiver { )?; } } - output.push_str("\n"); + output.push('\n'); } } } } - output.push_str("\n"); + output.push('\n'); } "li" => { - // Handle unordered list items if depth == 0 { output.push_str("- "); } @@ -387,14 +424,13 @@ impl TelegraphArchiver { } } if depth == 0 { - output.push_str("\n"); + output.push('\n'); } } "hr" => { output.push_str("---\n\n"); } "figure" => { - // Extract caption from figcaption first let mut caption = String::new(); if let Some(children) = &element.children { for child in children { @@ -411,8 +447,6 @@ impl TelegraphArchiver { } } } - - // Now process all children with the caption context if let Some(children) = &element.children { for child in children { if let Node::Element(elem) = child { @@ -432,11 +466,10 @@ impl TelegraphArchiver { } } } - output.push_str("\n"); + output.push('\n'); } "figcaption" => { - // Figcaptions are now handled by the figure element above - // Skip processing them directly to avoid duplication + // handled by parent
} "iframe" | "video" => { if let Some(attrs) = &element.attrs { @@ -452,7 +485,6 @@ impl TelegraphArchiver { } } "aside" => { - // Convert aside to blockquote output.push_str("> "); if let Some(children) = &element.children { for child in children { @@ -467,7 +499,6 @@ impl TelegraphArchiver { output.push_str("\n\n"); } _ => { - // For unhandled tags, just process children if let Some(children) = &element.children { for child in children { self.convert_node_to_markdown_with_context( @@ -485,39 +516,624 @@ impl TelegraphArchiver { Ok(()) } - fn clean_excessive_newlines(&self, content: &str) -> String { - // Replace 3+ consecutive newlines with just 2 - let mut result = content.to_string(); - while result.contains("\n\n\n") { - result = result.replace("\n\n\n", "\n\n"); + // ── Generic HTML path ───────────────────────────────────────────────────── + + async fn archive_generic_url(&self, url: &str) -> Result> { + let page = self.fetch_generic_page(url).await?; + let markdown = self.convert_generic_to_markdown(&page); + let filename = self.generate_generic_filename(&page); + let file_path = format!("content/{}", filename); + fs::write(&file_path, markdown)?; + let nonograph_id = filename.trim_end_matches(".md"); + Ok(format!("/{}", nonograph_id)) + } + + async fn fetch_generic_page( + &self, + url: &str, + ) -> Result> { + let client = reqwest::Client::builder() + .user_agent("Mozilla/5.0 (compatible; nonograph-archiver/1.0)") + .build()?; + + let response = client.get(url).send().await?; + if !response.status().is_success() { + return Err(format!("HTTP {}", response.status()).into()); } - result + let html = response.text().await?; + self.parse_generic_html(url, &html) + } + + fn parse_generic_html( + &self, + url: &str, + html: &str, + ) -> Result> { + let document = Html::parse_document(html); + + // ── Title ───────────────────────────────────────────────────────────── + // Priority: og:title > > h1 + let title = self + .select_attr(&document, "meta[property='og:title']", "content") + .or_else(|| self.select_attr(&document, "meta[name='twitter:title']", "content")) + .or_else(|| self.select_text(&document, "title")) + .or_else(|| self.select_text(&document, "h1")) + .unwrap_or_else(|| "Untitled".to_string()); + + let title = title.trim().to_string(); + + // ── Author ──────────────────────────────────────────────────────────── + let author = self + .select_attr(&document, "meta[name='author']", "content") + .or_else(|| self.select_attr(&document, "meta[property='article:author']", "content")) + .or_else(|| self.select_text(&document, "[rel='author']")) + .or_else(|| self.select_text(&document, ".author")) + .or_else(|| self.select_text(&document, ".byline")) + .or_else(|| self.select_text(&document, "[itemprop='author']")); + + // ── Date ────────────────────────────────────────────────────────────── + let date = self + .select_attr( + &document, + "meta[property='article:published_time']", + "content", + ) + .or_else(|| self.select_attr(&document, "meta[name='date']", "content")) + .or_else(|| self.select_attr(&document, "time", "datetime")) + .or_else(|| self.select_text(&document, "time")) + .map(|d| self.format_date_string(&d)); + + // ── Content ─────────────────────────────────────────────────────────── + // Try common article containers, fall back to <body> + let content_selectors = [ + "article", + "[role='main']", + "main", + ".post-content", + ".article-content", + ".entry-content", + ".story-body", + ".postbody", + "#content", + ".content", + ]; + + let content_html = content_selectors + .iter() + .find_map(|sel| { + Selector::parse(sel) + .ok() + .and_then(|s| document.select(&s).next().map(|el| el.html())) + }) + .or_else(|| { + Selector::parse("body") + .ok() + .and_then(|s| document.select(&s).next().map(|el| el.html())) + }) + .unwrap_or_default(); + + Ok(GenericPage { + url: url.to_string(), + title, + author, + date, + content_html, + }) + } + + /// Convert a `GenericPage` to Nonograph-flavoured markdown. + fn convert_generic_to_markdown(&self, page: &GenericPage) -> String { + let now: DateTime<Utc> = Utc::now(); + let archival_date = page + .date + .clone() + .unwrap_or_else(|| now.format("%B %d, %Y").to_string()); + + let mut md = String::new(); + + // Header line + if let Some(author) = &page.author { + let author = author.trim(); + if !author.is_empty() { + md.push_str(&format!("{} | {}\n\n", archival_date, author)); + } else { + md.push_str(&format!("{}\n\n", archival_date)); + } + } else { + md.push_str(&format!("{}\n\n", archival_date)); + } + + // Title + md.push_str(&format!("# {}\n", page.title)); + + // Source comment + md.push_str(&format!("// Source: {}\n\n", page.url)); + + // Body + let body = self.html_to_markdown(&page.content_html, &page.url); + md.push_str(&body); + + self.clean_excessive_newlines(&md) } + /// Walk an HTML fragment and emit Nonograph markdown. + fn html_to_markdown(&self, html: &str, base_url: &str) -> String { + let fragment = Html::parse_fragment(html); + let mut out = String::new(); + // scraper gives us an implicit root element; iterate its children + for node in fragment.root_element().children() { + self.walk_node(node, &mut out, base_url, 0); + } + self.clean_excessive_newlines(&out) + } + + fn walk_node( + &self, + node: NodeRef<ScraperNode>, + out: &mut String, + base_url: &str, + depth: usize, + ) { + match node.value() { + ScraperNode::Text(text) => { + let t = text.trim(); + if !t.is_empty() { + out.push_str(t); + } + } + ScraperNode::Element(el) => { + let tag = el.name(); + + // Skip non-content elements entirely + match tag { + "script" | "style" | "noscript" | "nav" | "header" | "footer" | "aside" + | "form" | "button" | "input" | "select" | "textarea" | "label" | "svg" + | "canvas" => return, + _ => {} + } + + match tag { + // ── Block elements ──────────────────────────────────────── + "p" | "div" | "section" | "article" | "main" => { + let inner = self.walk_children_to_string(node, out, base_url, depth); + if !inner.trim().is_empty() { + out.push_str(inner.trim()); + out.push_str("\n\n"); + } + } + "h1" => { + let inner = self.children_text(node, base_url, depth); + if !inner.trim().is_empty() { + out.push_str(&format!("# {}\n\n", inner.trim())); + } + } + "h2" => { + let inner = self.children_text(node, base_url, depth); + if !inner.trim().is_empty() { + out.push_str(&format!("## {}\n\n", inner.trim())); + } + } + "h3" => { + let inner = self.children_text(node, base_url, depth); + if !inner.trim().is_empty() { + out.push_str(&format!("### {}\n\n", inner.trim())); + } + } + "h4" | "h5" | "h6" => { + let inner = self.children_text(node, base_url, depth); + if !inner.trim().is_empty() { + out.push_str(&format!("#### {}\n\n", inner.trim())); + } + } + "blockquote" => { + let inner = self.children_text(node, base_url, depth); + for line in inner.trim().lines() { + out.push_str(&format!("> {}\n", line)); + } + out.push('\n'); + } + "pre" => { + let inner = self.children_text(node, base_url, depth); + // Try to detect language from a class like "language-rust" + let lang = el + .attr("class") + .and_then(|c| { + c.split_whitespace().find_map(|cls| { + cls.strip_prefix("language-") + .or_else(|| cls.strip_prefix("lang-")) + }) + }) + .unwrap_or(""); + out.push_str(&format!("```{}\n{}\n```\n\n", lang, inner.trim())); + } + "hr" => { + out.push_str("---\n\n"); + } + "br" => { + out.push('\n'); + } + "ul" => { + for child in node.children() { + if let ScraperNode::Element(li_el) = child.value() { + if li_el.name() == "li" { + let inner = self.children_text(child, base_url, depth + 1); + out.push_str(&format!("- {}\n", inner.trim())); + } + } + } + out.push('\n'); + } + "ol" => { + let mut idx = 1usize; + for child in node.children() { + if let ScraperNode::Element(li_el) = child.value() { + if li_el.name() == "li" { + let inner = self.children_text(child, base_url, depth + 1); + out.push_str(&format!("{}. {}\n", idx, inner.trim())); + idx += 1; + } + } + } + out.push('\n'); + } + "li" => { + // Standalone <li> outside of ul/ol + let inner = self.children_text(node, base_url, depth); + out.push_str(&format!("- {}\n", inner.trim())); + } + "figure" => { + // Look for img/video + figcaption + let mut img_md = String::new(); + let mut caption = String::new(); + for child in node.children() { + if let ScraperNode::Element(cel) = child.value() { + match cel.name() { + "figcaption" => { + caption = self.children_text(child, base_url, depth); + } + "img" => { + if let Some(src) = cel.attr("src") { + let src = self.resolve_url(src, base_url); + img_md = format!("![placeholder]({})", src); + } + } + "video" => { + if let Some(src) = cel.attr("src") { + let src = self.resolve_url(src, base_url); + img_md = format!("![placeholder]({})", src); + } + } + _ => {} + } + } + } + if !img_md.is_empty() { + // Replace placeholder with actual caption + let final_md = img_md.replace("placeholder", caption.trim()); + out.push_str(&final_md); + out.push_str("\n\n"); + } + } + + // ── Inline elements ─────────────────────────────────────── + "strong" | "b" => { + let inner = self.children_text(node, base_url, depth); + if !inner.trim().is_empty() { + out.push_str(&format!("**{}**", inner.trim())); + } + } + "em" | "i" => { + let inner = self.children_text(node, base_url, depth); + if !inner.trim().is_empty() { + out.push_str(&format!("*{}*", inner.trim())); + } + } + "u" => { + let inner = self.children_text(node, base_url, depth); + if !inner.trim().is_empty() { + out.push_str(&format!("_{}_", inner.trim())); + } + } + "s" | "del" | "strike" => { + let inner = self.children_text(node, base_url, depth); + if !inner.trim().is_empty() { + out.push_str(&format!("~{}~", inner.trim())); + } + } + "code" => { + let inner = self.children_text(node, base_url, depth); + if !inner.trim().is_empty() { + out.push_str(&format!("`{}`", inner.trim())); + } + } + "mark" => { + let inner = self.children_text(node, base_url, depth); + if !inner.trim().is_empty() { + out.push_str(&format!("=={} ==", inner.trim())); + } + } + "sup" => { + let inner = self.children_text(node, base_url, depth); + if !inner.trim().is_empty() { + out.push_str(&format!("^{}^", inner.trim())); + } + } + "a" => { + let href = el + .attr("href") + .map(|h| self.resolve_url(h, base_url)) + .unwrap_or_default(); + let inner = self.children_text(node, base_url, depth); + let text = inner.trim(); + if href.is_empty() { + out.push_str(text); + } else if text.is_empty() || text == href { + out.push_str(&format!("[{}]", href)); + } else { + out.push_str(&format!("[{}]({})", text, href)); + } + } + "img" => { + let src = el + .attr("src") + .map(|s| self.resolve_url(s, base_url)) + .unwrap_or_default(); + if !src.is_empty() { + let alt = el.attr("alt").unwrap_or(""); + out.push_str(&format!("![{}]({})", alt, src)); + } + } + "video" => { + let src = el + .attr("src") + .map(|s| self.resolve_url(s, base_url)) + .unwrap_or_default(); + if !src.is_empty() { + out.push_str(&format!("![Video]({})\n\n", src)); + } else { + // Try <source> children + for child in node.children() { + if let ScraperNode::Element(cel) = child.value() { + if cel.name() == "source" { + if let Some(s) = cel.attr("src") { + let s = self.resolve_url(s, base_url); + out.push_str(&format!("![Video]({})\n\n", s)); + break; + } + } + } + } + } + } + "iframe" => { + if let Some(src) = el.attr("src") { + let src = self.resolve_url(src, base_url); + // Only keep YouTube embeds etc. as links + if !src.is_empty() { + out.push_str(&format!("[{}]({})\n\n", src, src)); + } + } + } + "table" => { + self.table_to_markdown(node, out, base_url, depth); + } + _ => { + // Pass-through: just process children + for child in node.children() { + self.walk_node(child, out, base_url, depth); + } + } + } + } + _ => {} + } + } + + /// Walk children, accumulating inline output into a new String and also + /// pushing block-level output directly to `out`. Returns inline text. + fn walk_children_to_string( + &self, + node: NodeRef<ScraperNode>, + out: &mut String, + base_url: &str, + depth: usize, + ) -> String { + let mut inner = String::new(); + for child in node.children() { + self.walk_node(child, &mut inner, base_url, depth); + } + // Flush any block-level content that ended up in inner to out + // (walk_node already pushes directly; this just collects inline text) + let _ = out; // used in some code paths above; keep signature symmetric + inner + } + + /// Collect all text content under a node (inline only, no block wrappers). + fn children_text(&self, node: NodeRef<ScraperNode>, base_url: &str, depth: usize) -> String { + let mut s = String::new(); + for child in node.children() { + self.walk_node(child, &mut s, base_url, depth); + } + s + } + + /// Rudimentary table → Nonograph markdown table converter. + fn table_to_markdown( + &self, + node: NodeRef<ScraperNode>, + out: &mut String, + base_url: &str, + depth: usize, + ) { + let mut rows: Vec<Vec<String>> = Vec::new(); + let mut is_header_row: Vec<bool> = Vec::new(); + + for child in node.descendants() { + if let ScraperNode::Element(el) = child.value() { + match el.name() { + "tr" => { + let mut cells: Vec<String> = Vec::new(); + let mut has_th = false; + for td in child.children() { + if let ScraperNode::Element(cel) = td.value() { + match cel.name() { + "th" => { + has_th = true; + cells.push( + self.children_text(td, base_url, depth) + .trim() + .to_string(), + ); + } + "td" => { + cells.push( + self.children_text(td, base_url, depth) + .trim() + .to_string(), + ); + } + _ => {} + } + } + } + if !cells.is_empty() { + is_header_row.push(has_th); + rows.push(cells); + } + } + _ => {} + } + } + } + + if rows.is_empty() { + return; + } + + // Determine column count from widest row + let col_count = rows.iter().map(|r| r.len()).max().unwrap_or(0); + + for (i, row) in rows.iter().enumerate() { + let padded: Vec<String> = (0..col_count) + .map(|j| row.get(j).cloned().unwrap_or_default()) + .collect(); + out.push_str(&format!("| {} |\n", padded.join(" | "))); + // Insert separator after the first row (header) + if i == 0 { + let sep: Vec<&str> = (0..col_count).map(|_| "---").collect(); + out.push_str(&format!("|{}|\n", sep.join("|"))); + } + } + out.push('\n'); + } + + // ── URL helpers ─────────────────────────────────────────────────────────── + + /// Resolve a potentially relative URL against the page's base URL. + fn resolve_url(&self, href: &str, base_url: &str) -> String { + if href.starts_with("http://") || href.starts_with("https://") || href.starts_with("//") { + return href.to_string(); + } + if let Ok(base) = url::Url::parse(base_url) { + if let Ok(resolved) = base.join(href) { + return resolved.to_string(); + } + } + href.to_string() + } + + // ── CSS selector helpers ────────────────────────────────────────────────── + + fn select_text(&self, document: &Html, selector: &str) -> Option<String> { + let sel = Selector::parse(selector).ok()?; + document + .select(&sel) + .next() + .map(|el| el.text().collect::<String>().trim().to_string()) + .filter(|s| !s.is_empty()) + } + + fn select_attr(&self, document: &Html, selector: &str, attr: &str) -> Option<String> { + let sel = Selector::parse(selector).ok()?; + document + .select(&sel) + .next() + .and_then(|el| el.value().attr(attr)) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + } + + // ── Date normalisation ──────────────────────────────────────────────────── + + /// Try to turn an ISO-8601 or other date string into "Month DD, YYYY". + fn format_date_string(&self, raw: &str) -> String { + let trimmed = raw.trim(); + // Try full ISO datetime first + if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(trimmed) { + return dt.format("%B %d, %Y").to_string(); + } + // Try date-only YYYY-MM-DD + if let Ok(d) = chrono::NaiveDate::parse_from_str(trimmed, "%Y-%m-%d") { + return d.format("%B %d, %Y").to_string(); + } + // Return as-is if we can't parse it + trimmed.to_string() + } + + // ── Filename generation ─────────────────────────────────────────────────── + fn generate_filename(&self, page: &TelegraphPage) -> String { - // Use the path from Telegraph as base, but make it filesystem-safe let mut filename = page.path.clone(); - - // Replace any unsafe characters - filename = filename.replace('/', "-"); - filename = filename.replace('\\', "-"); - filename = filename.replace(':', "-"); - filename = filename.replace('?', "-"); - filename = filename.replace('*', "-"); - filename = filename.replace('"', "-"); - filename = filename.replace('<', "-"); - filename = filename.replace('>', "-"); - filename = filename.replace('|', "-"); - - // Ensure it ends with .md + for ch in &['/', '\\', ':', '?', '*', '"', '<', '>', '|'] { + filename = filename.replace(*ch, "-"); + } if !filename.ends_with(".md") { filename.push_str(".md"); } - filename } + + /// Derive a filesystem-safe filename from a generic page. + fn generate_generic_filename(&self, page: &GenericPage) -> String { + let now: DateTime<Utc> = Utc::now(); + let date_suffix = now.format("%m-%d-%Y").to_string(); + + // Slugify the title + let slug: String = page + .title + .to_lowercase() + .chars() + .map(|c| if c.is_alphanumeric() { c } else { '-' }) + .collect::<String>() + // Collapse runs of dashes + .split('-') + .filter(|s| !s.is_empty()) + .collect::<Vec<_>>() + .join("-"); + + // Truncate slug so the total filename stays reasonable + let slug = if slug.len() > 60 { + slug[..60].trim_end_matches('-').to_string() + } else { + slug + }; + + format!("{}-{}.md", slug, date_suffix) + } + + // ── Shared utilities ────────────────────────────────────────────────────── + + fn clean_excessive_newlines(&self, content: &str) -> String { + let mut result = content.to_string(); + while result.contains("\n\n\n") { + result = result.replace("\n\n\n", "\n\n"); + } + result + } } +// ─── Tests ──────────────────────────────────────────────────────────────────── + #[cfg(test)] mod tests { use super::*; @@ -552,4 +1168,145 @@ mod tests { let filename = archiver.generate_filename(&page); assert_eq!(filename, "Sample-Page-12-15.md"); } + + #[test] + fn test_is_telegraph_url_routed_correctly() { + // Just confirm the routing logic — no network needed + let telegraph = "https://telegra.ph/My-Post-01-01"; + let generic = "https://www.example.com/article/foo"; + + let t_parsed = url::Url::parse(telegraph).unwrap(); + let g_parsed = url::Url::parse(generic).unwrap(); + + assert_eq!(t_parsed.host_str(), Some("telegra.ph")); + assert_ne!(g_parsed.host_str(), Some("telegra.ph")); + } + + #[test] + fn test_generate_generic_filename_slug() { + let archiver = TelegraphArchiver::new(); + let page = GenericPage { + url: "https://example.com/test".to_string(), + title: "Hello World! This Is A Test".to_string(), + author: None, + date: None, + content_html: String::new(), + }; + let filename = archiver.generate_generic_filename(&page); + // Should start with the slugified title + assert!(filename.starts_with("hello-world-this-is-a-test-")); + assert!(filename.ends_with(".md")); + } + + #[test] + fn test_resolve_url_absolute() { + let archiver = TelegraphArchiver::new(); + let result = archiver.resolve_url( + "https://cdn.example.com/img.png", + "https://example.com/page", + ); + assert_eq!(result, "https://cdn.example.com/img.png"); + } + + #[test] + fn test_resolve_url_relative() { + let archiver = TelegraphArchiver::new(); + let result = archiver.resolve_url("/images/photo.jpg", "https://example.com/article/1"); + assert_eq!(result, "https://example.com/images/photo.jpg"); + } + + #[test] + fn test_format_date_string_iso() { + let archiver = TelegraphArchiver::new(); + let result = archiver.format_date_string("2026-04-30T17:59:01+00:00"); + assert_eq!(result, "April 30, 2026"); + } + + #[test] + fn test_format_date_string_date_only() { + let archiver = TelegraphArchiver::new(); + let result = archiver.format_date_string("2026-04-30"); + assert_eq!(result, "April 30, 2026"); + } + + #[test] + fn test_html_to_markdown_basic() { + let archiver = TelegraphArchiver::new(); + let html = "<p>Hello <strong>world</strong>!</p>"; + let md = archiver.html_to_markdown(html, "https://example.com"); + assert!(md.contains("**world**")); + assert!(md.contains("Hello")); + } + + #[test] + fn test_html_to_markdown_headings() { + let archiver = TelegraphArchiver::new(); + let html = "<h2>Section Title</h2><p>Some text.</p>"; + let md = archiver.html_to_markdown(html, "https://example.com"); + assert!(md.contains("## Section Title")); + assert!(md.contains("Some text.")); + } + + #[test] + fn test_html_to_markdown_links() { + let archiver = TelegraphArchiver::new(); + let html = r#"<p><a href="https://example.com">Click here</a></p>"#; + let md = archiver.html_to_markdown(html, "https://example.com"); + assert!(md.contains("[Click here](https://example.com)")); + } + + #[test] + fn test_html_to_markdown_blockquote() { + let archiver = TelegraphArchiver::new(); + let html = "<blockquote>Quoted text here.</blockquote>"; + let md = archiver.html_to_markdown(html, "https://example.com"); + assert!(md.contains("> Quoted text here.")); + } + + #[test] + fn test_html_to_markdown_lists() { + let archiver = TelegraphArchiver::new(); + let html = "<ul><li>Item one</li><li>Item two</li></ul>"; + let md = archiver.html_to_markdown(html, "https://example.com"); + assert!(md.contains("- Item one")); + assert!(md.contains("- Item two")); + } + + #[test] + fn test_parse_generic_html_title() { + let archiver = TelegraphArchiver::new(); + let html = r#"<html><head> + <meta property="og:title" content="OG Title"> + <title>Page Title +

Content

"#; + let page = archiver + .parse_generic_html("https://example.com", html) + .unwrap(); + assert_eq!(page.title, "OG Title"); + } + + #[test] + fn test_parse_generic_html_author() { + let archiver = TelegraphArchiver::new(); + let html = r#" + +

Content

+ "#; + let page = archiver + .parse_generic_html("https://example.com", html) + .unwrap(); + assert_eq!(page.author.as_deref(), Some("Jane Doe")); + } + + #[test] + fn test_parse_generic_html_date_iso() { + let archiver = TelegraphArchiver::new(); + let html = r#" + +

Content

"#; + let page = archiver + .parse_generic_html("https://example.com", html) + .unwrap(); + assert_eq!(page.date.as_deref(), Some("April 30, 2026")); + } } diff --git a/src/parser.rs b/src/parser.rs index 57abcbb..a5bffc0 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -391,7 +391,7 @@ fn sanitize_html(html: String) -> String { .add_tag_attributes("th", &["style"]) .add_tag_attributes("td", &["style"]) .add_tag_attributes("a", &["href", "target", "id", "class"]) - .add_tag_attributes("div", &["class"]) + .add_tag_attributes("div", &["class", "style"]) .add_tag_attributes("hr", &["class"]) .add_tag_attributes("li", &["id"]) .add_tag_attributes("sup", &["id"]) @@ -488,8 +488,90 @@ fn process_single_header(text: &str, header_count: &mut usize) -> Option } fn process_single_blockquote(text: &str) -> String { - let mut blockquote_content = String::new(); + const ALERT_TYPES: &[&str] = &["NOTE", "TIP", "IMPORTANT", "WARNING", "CAUTION"]; + + // Collect all "> " lines in order + let quote_lines: Vec<&str> = text + .lines() + .map(|l| l.trim()) + .filter(|l| l.starts_with("> ")) + .map(|l| &l[2..]) + .collect(); + + // Check whether the first line is a GitHub-style alert marker + if let Some(&first) = quote_lines.first() { + let upper = first.trim().to_uppercase(); + if let Some(alert_type) = ALERT_TYPES.iter().find(|&&t| format!("[!{}]", t) == upper) { + let lowercase = alert_type.to_lowercase(); + let title_case = { + let mut chars = alert_type.chars(); + match chars.next() { + None => String::new(), + Some(c) => c.to_uppercase().to_string() + &chars.as_str().to_lowercase(), + } + }; + + let body = quote_lines[1..].join("
"); + + return format!( + "
\n
{title_case}
\n
{body}
\n
", + ); + } + } + if let Some(&first) = quote_lines.first() { + let trimmed_first = first.trim(); + if trimmed_first.starts_with('[') && trimmed_first.ends_with(']') { + let inner = &trimmed_first[1..trimmed_first.len() - 1]; + if let Some(bang_pos) = inner.find('!') { + let hex_part = &inner[..bang_pos]; + let label_part = &inner[bang_pos + 1..]; + + let hex_len = hex_part.len(); + let is_valid_hex = (hex_len == 3 || hex_len == 6) + && hex_part.chars().all(|c| c.is_ascii_hexdigit()); + + if is_valid_hex { + let sanitized: String = label_part + .chars() + .filter(|c| { + c.is_ascii_alphanumeric() || *c == ' ' || *c == '-' || *c == '_' + }) + .collect(); + let sanitized = sanitized.trim(); + let sanitized: String = sanitized.chars().take(25).collect(); + + if !sanitized.is_empty() { + let hex6: String = if hex_len == 3 { + hex_part.chars().flat_map(|c| [c, c]).collect() + } else { + hex_part.to_string() + }; + let hex6_upper = hex6.to_uppercase(); + + let r = u8::from_str_radix(&hex6_upper[0..2], 16).unwrap_or(0); + let g = u8::from_str_radix(&hex6_upper[2..4], 16).unwrap_or(0); + let b = u8::from_str_radix(&hex6_upper[4..6], 16).unwrap_or(0); + + let escaped_label = sanitized + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """); + + let body = quote_lines[1..].join("
"); + + return format!( + "
\n
{escaped_label}
\n
{body}
\n
", + ); + } + } + } + } + } + + // Fall back to plain blockquote + let mut blockquote_content = String::new(); for line in text.lines() { let trimmed = line.trim(); if trimmed.starts_with("> ") { @@ -3623,4 +3705,228 @@ Final paragraph with normal text."#; assert!(multiple_result.contains("
")); assert!(multiple_result.contains("
")); } + + #[test] + fn test_alert_blockquotes() { + // Note alert + let text = "> [!NOTE]\n> Useful information."; + let result = render_markdown(text); + assert!(result.contains("class=\"alert alert-note\"")); + assert!(result.contains("class=\"alert-label\"")); + assert!(result.contains("Note")); + assert!(result.contains("Useful information.")); + assert!(!result.contains("[!NOTE]")); + + // Warning alert + let text = "> [!WARNING]\n> Be careful!"; + let result = render_markdown(text); + assert!(result.contains("class=\"alert alert-warning\"")); + assert!(result.contains("Warning")); + assert!(result.contains("Be careful!")); + + // Non-alert blockquote still works + let text = "> Just a regular quote"; + let result = render_markdown(text); + assert!(result.contains("
Just a regular quote
")); + + // Case-insensitive detection + let text = "> [!tip]\n> Try this."; + let result = render_markdown(text); + assert!(result.contains("class=\"alert alert-tip\"")); + assert!(result.contains("Tip")); + + // All five types render correctly + for (marker, class, label) in &[ + ("NOTE", "alert-note", "Note"), + ("TIP", "alert-tip", "Tip"), + ("IMPORTANT", "alert-important", "Important"), + ("WARNING", "alert-warning", "Warning"), + ("CAUTION", "alert-caution", "Caution"), + ] { + let text = format!("> [!{}]\n> Body text.", marker); + let result = render_markdown(&text); + assert!( + result.contains(&format!("class=\"alert {}\"", class)), + "missing class for {}", + marker + ); + assert!( + result.contains(&format!("class=\"alert-label\">{}", label)), + "missing label for {}", + marker + ); + assert!(result.contains("Body text."), "missing body for {}", marker); + assert!( + !result.contains(&format!("[!{}]", marker)), + "marker leaked into output for {}", + marker + ); + } + + // Unknown alert type falls back to plain blockquote + let text = "> [!UNKNOWN]\n> Some text."; + let result = render_markdown(text); + assert!( + result.contains("
"), + "unknown type should fall back to blockquote" + ); + assert!( + !result.contains("class=\"alert\""), + "unknown type should not produce an alert" + ); + + // Marker-only alert (no body) renders without crashing + let text = "> [!NOTE]"; + let result = render_markdown(text); + assert!(result.contains("class=\"alert alert-note\"")); + assert!(result.contains("class=\"alert-body\"")); + } + + #[test] + fn test_alert_inline_formatting() { + // Bold inside alert body + let text = "> [!NOTE]\n> This is **bold** text."; + let result = render_markdown(text); + assert!( + result.contains("bold"), + "bold should render inside alert" + ); + + // Italic inside alert body + let text = "> [!TIP]\n> This is *italic* text."; + let result = render_markdown(text); + assert!( + result.contains("italic"), + "italic should render inside alert" + ); + + // Underline inside alert body + let text = "> [!IMPORTANT]\n> This is _underlined_ text."; + let result = render_markdown(text); + assert!( + result.contains("underlined"), + "underline should render inside alert" + ); + + // Strikethrough inside alert body + let text = "> [!WARNING]\n> This is ~struck~ text."; + let result = render_markdown(text); + assert!( + result.contains("struck"), + "strikethrough should render inside alert" + ); + + // Inline code inside alert body + let text = "> [!CAUTION]\n> Use `rm -rf` carefully."; + let result = render_markdown(text); + assert!( + result.contains("rm -rf"), + "inline code should render inside alert" + ); + + // Link inside alert body + let text = "> [!NOTE]\n> See [example](https://example.com)."; + let result = render_markdown(text); + assert!(result.contains("highlighted"), + "highlight should render inside alert" + ); + + // Multiple formatting on multiple lines + let text = "> [!IMPORTANT]\n> **First** line.\n> *Second* line."; + let result = render_markdown(text); + assert!( + result.contains("First"), + "bold on first body line" + ); + assert!( + result.contains("Second"), + "italic on second body line" + ); + assert!( + result.contains("
"), + "body lines should be joined with
" + ); + } + + #[test] + fn test_alert_security() { + // XSS via alert body — script tag should be stripped + let text = "> [!NOTE]\n> Safe text."; + let result = render_markdown(text); + assert!( + !result.contains(""), + "closing script tag must be stripped" + ); + + // XSS via alert type marker — crafted marker should not inject HTML + let text = "> [!NOTE\">]\n> Body."; + let result = render_markdown(text); + assert!( + !result.contains("