feat: tag topics and lessons to CCSS focus standards; load grades 7-8

config/application.rb

@@ -12,7 +12,7 @@
 require "action_text/engine"
 require "action_view/railtie"
 require "action_cable/engine"
-# require "rails/test_unit/railtie"
+require "rails/test_unit/railtie"
 
 # Require the gems listed in Gemfile, including any gems
 # you've limited to :test, :development, or :production.

lib/engageny/focus_standard_tagger.rb

@@ -0,0 +1,124 @@
+require "nokogiri"
+
+module Engageny
+  # Extracts CCSS codes from the "Focus Standards" markup EngageNY topic
+  # overviews carry (a `ny-list-focusstandards` CSS class applied to the
+  # paragraphs/list items Aspose emits for that Word list style, the same
+  # convention as `ny-h1` for headings — see Engageny::Importer#extract_topic_title)
+  # and turns them into StandardTagging rows.
+  #
+  # Focus Standards are listed once per Topic overview in the source
+  # curriculum, not per Lesson, so the Topic is the canonical taggable and
+  # every Lesson under it inherits the union of its Topic's standards.
+  class FocusStandardTagger
+    FOCUS_STANDARDS_CLASS = "ny-list-focusstandards"
+
+    # Matches a full standard code, optionally with an embedded cluster
+    # letter (e.g. "5.NBT.1" or "5.NBT.A.1"), or a bare cluster reference
+    # with no leaf number (e.g. "5.NBT.A"). The longer, number-bearing form
+    # is tried first so it wins over the shorter cluster-only alternative.
+    CODE_PATTERN = %r{
+      \d{1,2}\.[A-Z]{1,4}\.(?:[A-Z]\.)?\d+[a-z]?  # e.g. 5.NBT.1, 5.NBT.A.1, 5.NF.4a
+      | \d{1,2}\.[A-Z]{1,4}\.[A-Z]\b                # cluster only, e.g. 5.NBT.A
+    }x
+
+    Report = Struct.new(:topic_taggings, :lesson_taggings, :unresolved, keyword_init: true) do
+      def initialize(**kwargs)
+        super(topic_taggings: 0, lesson_taggings: 0, unresolved: Hash.new(0), **kwargs)
+      end
+
+      def merge!(other)
+        self.topic_taggings += other.topic_taggings
+        self.lesson_taggings += other.lesson_taggings
+        other.unresolved.each { |code, count| unresolved[code] += count }
+        self
+      end
+    end
+
+    # Extracts the raw CCSS codes present in the given HTML's focus-standards
+    # markup. Returns [] for blank, nil, or malformed HTML, or HTML with no
+    # focus-standards markup at all. Never raises.
+    def self.extract_codes(html)
+      return [] if html.blank?
+
+      doc = Nokogiri::HTML::DocumentFragment.parse(html)
+      nodes = doc.css(".#{FOCUS_STANDARDS_CLASS}")
+      return [] if nodes.empty?
+
+      text = nodes.map(&:text).join(" ")
+      text = text.scrub("")
+      text.scan(CODE_PATTERN).uniq
+    rescue StandardError
+      []
+    end
+
+    # Normalizes a raw extracted code to the dotted form Standard#code uses:
+    # grade.domain.number[subletter], with any embedded cluster letter
+    # stripped (our imported Standard rows never carry the cluster letter,
+    # e.g. "5.NBT.1" not "5.NBT.A.1"). A bare cluster reference like
+    # "5.NBT.A" has no leaf number to normalize to and is returned as-is —
+    # it will not resolve to a Standard and is reported unresolved.
+    def self.normalize(raw_code)
+      parts = raw_code.split(".")
+      return raw_code unless parts.last.match?(/\A\d+[a-z]?\z/)
+      return raw_code unless parts.length == 4
+
+      [ parts[0], parts[1], parts[3] ].join(".")
+    end
+
+    # Tags a single Topic (from its overview_html) and every Lesson beneath
+    # it (with the same, unioned set of standards). Idempotent: existing
+    # StandardTagging rows are left alone and not recreated.
+    def self.tag_topic!(topic)
+      report = Report.new
+      codes = extract_codes(topic.overview_html)
+      return report if codes.empty?
+
+      standards = []
+      codes.each do |raw|
+        normalized = normalize(raw)
+        standard = Standard.find_by(code: normalized)
+        if standard
+          standards << standard
+        else
+          report.unresolved[normalized] += 1
+        end
+      end
+      standards.uniq!
+      return report if standards.empty?
+
+      report.topic_taggings += create_missing_taggings(topic, standards)
+
+      topic.lessons.each do |lesson|
+        report.lesson_taggings += create_missing_taggings(lesson, standards)
+      end
+
+      report
+    end
+
+    def self.create_missing_taggings(taggable, standards)
+      existing_ids = StandardTagging.where(
+        taggable_type: taggable.class.polymorphic_name,
+        taggable_id: taggable.id
+      ).pluck(:standard_id)
+
+      to_create = standards.reject { |s| existing_ids.include?(s.id) }
+      return 0 if to_create.empty?
+
+      now = Time.current
+      StandardTagging.insert_all(
+        to_create.map do |standard|
+          {
+            standard_id: standard.id,
+            taggable_type: taggable.class.polymorphic_name,
+            taggable_id: taggable.id,
+            created_at: now,
+            updated_at: now
+          }
+        end
+      )
+      to_create.size
+    end
+    private_class_method :create_missing_taggings
+  end
+end

lib/tasks/ccss.rake

@@ -1,7 +1,7 @@
 namespace :ccss do
   desc "Import Common Core State Standards from JSON"
   task import: :environment do
-    path = Rails.root.join("data/ccss-math-grades-4-6.json")
+    path = Rails.root.join("data/ccss-math-grades-4-8.json")
     data = JSON.parse(File.read(path))
 
     created = 0

lib/tasks/standards.rake

@@ -0,0 +1,36 @@
+namespace :standards do
+  desc "Tag topics (and their lessons) with CCSS focus standards parsed from EngageNY overview HTML"
+  task tag_focus: :environment do
+    report = Engageny::FocusStandardTagger::Report.new
+    topics_with_html = 0
+    topics_without_html = 0
+
+    Topic.find_each do |topic|
+      if topic.overview_html.blank?
+        topics_without_html += 1
+        next
+      end
+
+      topics_with_html += 1
+      report.merge!(Engageny::FocusStandardTagger.tag_topic!(topic))
+    end
+
+    puts "Topics with overview HTML: #{topics_with_html}"
+    puts "Topics without overview HTML (skipped): #{topics_without_html}"
+    puts "Topic taggings created: #{report.topic_taggings}"
+    puts "Lesson taggings created: #{report.lesson_taggings}"
+
+    if report.unresolved.any?
+      puts "\nUnresolved codes (no matching Standard row):"
+      report.unresolved.sort_by { |_code, count| -count }.each do |code, count|
+        puts "  #{code} (#{count})"
+      end
+
+      report_path = Rails.root.join("tmp/standards_tag_focus_unresolved.txt")
+      File.write(report_path, report.unresolved.sort_by { |_code, count| -count }.map { |code, count| "#{code}\t#{count}" }.join("\n") + "\n")
+      puts "\nWrote unresolved-code report to #{report_path}"
+    else
+      puts "\nNo unresolved codes."
+    end
+  end
+end

mise.toml

@@ -0,0 +1,2 @@
+[tools]
+ruby = "3.4.4"

scripts/fetch_ccss_standards.rb

@@ -1,18 +1,18 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Fetches Common Core State Standards for Mathematics (Grades 4-6) from the
+# Fetches Common Core State Standards for Mathematics (Grades 4-8) from the
 # SirFizX/standards-data GitHub repo and structures them for CommonMath.
 #
 # Usage:
 #   ruby scripts/fetch_ccss_standards.rb
 #
-# Output: data/ccss-math-grades-4-6.json
+# Output: data/ccss-math-grades-4-8.json
 
 require "open-uri"
 require "json"
 
-OUTPUT_PATH = File.expand_path("../data/ccss-math-grades-4-6.json", __dir__)
+OUTPUT_PATH = File.expand_path("../data/ccss-math-grades-4-8.json", __dir__)
 SOURCE_URL = "https://raw.githubusercontent.com/SirFizX/standards-data/master/clean-data/CC/math/CC-math-0.8.0.json"
 
 DOMAIN_NAMES = {
@@ -24,11 +24,12 @@
   "RP" => "Ratios and Proportional Relationships",
   "NS" => "The Number System",
   "EE" => "Expressions and Equations",
-  "SP" => "Statistics and Probability"
+  "SP" => "Statistics and Probability",
+  "F" => "Functions"
 }.freeze
 
-TARGET_GRADES = %w[04 05 06].freeze
-TARGET_GRADE_INTS = [ 4, 5, 6 ].freeze
+TARGET_GRADES = %w[04 05 06 07 08].freeze
+TARGET_GRADE_INTS = [ 4, 5, 6, 7, 8 ].freeze
 
 $stderr.puts "Fetching CCSS data from #{SOURCE_URL}..."
 raw = URI.open(SOURCE_URL).read
@@ -49,7 +50,7 @@
   clusters[entry["code"]] = entry["statement"]
 end
 
-$stderr.puts "Found #{clusters.length} clusters for grades 4-6"
+$stderr.puts "Found #{clusters.length} clusters for grades #{TARGET_GRADE_INTS.first}-#{TARGET_GRADE_INTS.last}"
 
 # Second pass: extract leaf standards
 standards = []

test/fixtures/files/topic_overview_focus_standards.html

@@ -0,0 +1,10 @@
+<html>
+<head></head>
+<body>
+<p class="ny-h1">Topic A Place Value and Decimal Fractions</p>
+<p class="ny-normal">Focus Standard:</p>
+<p class="ny-list-focusstandards">5.NBT.1 Recognize that in a multi-digit number, a digit in one place represents 10 times as much as it represents in the place to its right and 1/10 of what it represents in the place to its left.</p>
+<p class="ny-list-focusstandards">5.NBT.2 Explain patterns in the number of zeros of the product when multiplying a number by powers of 10, and explain patterns in the placement of the decimal point when a decimal is multiplied or divided by a power of 10.</p>
+<p class="ny-normal">Instructional Days: 5</p>
+</body>
+</html>

test/fixtures/files/topic_overview_unresolved_focus_standard.html

@@ -0,0 +1,9 @@
+<html>
+<head></head>
+<body>
+<p class="ny-h1">Topic B Cluster and Out-of-Scope Codes</p>
+<p class="ny-normal">Focus Standard:</p>
+<p class="ny-list-focusstandards">5.NBT.A Understand the place value system.</p>
+<p class="ny-list-focusstandards">9.ZZ.1 Not a real grade-4-8 standard.</p>
+</body>
+</html>

test/lib/engageny/focus_standard_tagger_test.rb

@@ -0,0 +1,112 @@
+require "test_helper"
+
+module Engageny
+  class FocusStandardTaggerTest < ActiveSupport::TestCase
+    def focus_standards_html
+      file_fixture("topic_overview_focus_standards.html").read
+    end
+
+    def unresolved_focus_standard_html
+      file_fixture("topic_overview_unresolved_focus_standard.html").read
+    end
+
+    def build_topic(overview_html:, lesson_count: 2)
+      grade = Grade.create!(number: 5, title: "Grade 5")
+      content_module = grade.content_modules.create!(number: 1, title: "Module 1", position: 1)
+      topic = content_module.topics.create!(letter: "A", title: "Topic A", position: 1, overview_html: overview_html)
+      lesson_count.times do |i|
+        topic.lessons.create!(number: i + 1, position: i + 1)
+      end
+      topic
+    end
+
+    def create_standard(code:, grade_level: 5, domain: "Number and Operations in Base Ten")
+      Standard.create!(
+        code: code,
+        domain: domain,
+        description: "Description for #{code}",
+        grade_level: grade_level
+      )
+    end
+
+    test "extract_codes finds CCSS codes inside ny-list-focusstandards markup" do
+      codes = FocusStandardTagger.extract_codes(focus_standards_html)
+      assert_equal %w[5.NBT.1 5.NBT.2], codes.sort
+    end
+
+    test "extract_codes returns empty array for blank html" do
+      assert_equal [], FocusStandardTagger.extract_codes(nil)
+      assert_equal [], FocusStandardTagger.extract_codes("")
+    end
+
+    test "extract_codes returns empty array when the focus-standards class is absent" do
+      html = "<p class=\"ny-normal\">5.NBT.1 mentioned but not in the focus standards list</p>"
+      assert_equal [], FocusStandardTagger.extract_codes(html)
+    end
+
+    test "extract_codes yields nothing and does not raise on malformed markup" do
+      malformed = "<div class=\"ny-list-focusstandards\"><p>unterminated<div>"
+      assert_nothing_raised { FocusStandardTagger.extract_codes(malformed) }
+
+      binary_garbage = "<p class=\"ny-list-focusstandards\">\xFF\xFE not valid utf-8</p>".dup.force_encoding("ASCII-8BIT")
+      assert_nothing_raised { FocusStandardTagger.extract_codes(binary_garbage) }
+    end
+
+    test "normalize strips an embedded cluster letter but leaves bare cluster codes alone" do
+      assert_equal "5.NBT.1", FocusStandardTagger.normalize("5.NBT.1")
+      assert_equal "5.NBT.1", FocusStandardTagger.normalize("5.NBT.A.1")
+      assert_equal "5.NF.4a", FocusStandardTagger.normalize("5.NF.4a")
+      assert_equal "5.NBT.A", FocusStandardTagger.normalize("5.NBT.A")
+    end
+
+    test "tag_topic! creates a tagging on the topic and on every lesson beneath it" do
+      create_standard(code: "5.NBT.1")
+      create_standard(code: "5.NBT.2")
+      topic = build_topic(overview_html: focus_standards_html, lesson_count: 2)
+
+      report = FocusStandardTagger.tag_topic!(topic)
+
+      assert_equal 2, report.topic_taggings
+      assert_equal 4, report.lesson_taggings # 2 standards x 2 lessons
+      assert_empty report.unresolved
+
+      assert_equal %w[5.NBT.1 5.NBT.2], topic.standards.reload.pluck(:code).sort
+      topic.lessons.each do |lesson|
+        assert_equal %w[5.NBT.1 5.NBT.2], lesson.standards.reload.pluck(:code).sort
+      end
+    end
+
+    test "tag_topic! is idempotent on a second run" do
+      create_standard(code: "5.NBT.1")
+      create_standard(code: "5.NBT.2")
+      topic = build_topic(overview_html: focus_standards_html, lesson_count: 2)
+
+      FocusStandardTagger.tag_topic!(topic)
+      second_report = FocusStandardTagger.tag_topic!(topic)
+
+      assert_equal 0, second_report.topic_taggings
+      assert_equal 0, second_report.lesson_taggings
+      assert_equal 2, topic.standard_taggings.count
+    end
+
+    test "tag_topic! reports codes with no matching Standard row instead of dropping them" do
+      topic = build_topic(overview_html: unresolved_focus_standard_html, lesson_count: 1)
+
+      report = FocusStandardTagger.tag_topic!(topic)
+
+      assert_equal 0, report.topic_taggings
+      assert_equal 0, report.lesson_taggings
+      assert_equal({ "5.NBT.A" => 1, "9.ZZ.1" => 1 }, report.unresolved)
+    end
+
+    test "tag_topic! returns an empty report for a topic with no overview html" do
+      topic = build_topic(overview_html: nil, lesson_count: 1)
+
+      report = FocusStandardTagger.tag_topic!(topic)
+
+      assert_equal 0, report.topic_taggings
+      assert_equal 0, report.lesson_taggings
+      assert_empty report.unresolved
+    end
+  end
+end