add parse_audience to all taxila scrapers

config/tess.example.yml

@@ -164,6 +164,7 @@ default: &default
     #  biotools, topics, operations, sponsors, fairshare, county, ardc_fields_of_research,
     #  other_types, subsets, syllabus, approved_editors, address_finder
     disabled: ['ardc_fields_of_research', 'other_types', 'subsets', 'syllabus', 'approved_editors']
+    auto_parse_vars: [] # available features to auto parse from description: ['keywords', 'target_audience']
     materials_disabled: []
     content_providers_disabled: []
     bioschemas_testing: false

lib/ingestors/auto_parser_mappings/keywords.json

@@ -0,0 +1,140 @@
+{
+"Research data": "RDM, FAIR principles and open science",
+"Scientific disciplines": "RDM, FAIR principles and open science",
+"Research methodologies": "RDM, FAIR principles and open science",
+"Research data lifecycle": "RDM, FAIR principles and open science",
+"Roles and responsibilities in RDM": "RDM, FAIR principles and open science",
+"Open science": "RDM, FAIR principles and open science",
+"Reproducibility and replicability": "RDM, FAIR principles and open science",
+"Scholarly communication": "RDM, FAIR principles and open science",
+"Costs for RDM": "RDM, FAIR principles and open science",
+"Data management planning": "RDM, FAIR principles and open science",
+"FAIR data principles": "RDM, FAIR principles and open science",
+"Data discovery": "RDM, FAIR principles and open science",
+"Data reuse": "RDM, FAIR principles and open science",
+"Data collection": "RDM, FAIR principles and open science",
+"Data documentation": "RDM, FAIR principles and open science",
+"Data organisation": "RDM, FAIR principles and open science",
+"File naming (conventions)": "RDM, FAIR principles and open science",
+"Data versioning": "RDM, FAIR principles and open science",
+"Data formats and types": "RDM, FAIR principles and open science",
+"Data back-up": "RDM, FAIR principles and open science",
+"Data selection": "RDM, FAIR principles and open science",
+"Data destruction": "RDM, FAIR principles and open science",
+"Data preservation and archiving": "RDM, FAIR principles and open science",
+"Data publication": "RDM, FAIR principles and open science",
+"Data curation": "RDM, FAIR principles and open science",
+"Data visualisation": "RDM, FAIR principles and open science",
+"Data provenance": "RDM, FAIR principles and open science",
+"Metadata (standard)": "RDM, FAIR principles and open science",
+"Controlled vocabulary, ontology, taxonomy, thesaurus": "RDM, FAIR principles and open science",
+"Linked Open Data and SPARQL": "RDM, FAIR principles and open science",
+"FAIR metrics": "RDM, FAIR principles and open science",
+"3-point FAIRification Framework (FAIR data point, FAIR Implementation Profile)": "RDM, FAIR principles and open science",
+"Persistent identifier": "RDM, FAIR principles and open science",
+"Open Archives Initiative Protocol for Metadata Harvesting (OAI-PMH)": "RDM, FAIR principles and open science",
+"Spreadsheet tools": "RDM, FAIR principles and open science",
+"Data modelling": "RDM, FAIR principles and open science",
+"Data integration": "RDM, FAIR principles and open science",
+"Data integrity, validation & quality": "RDM, FAIR principles and open science",
+"Data cleaning & wrangling": "RDM, FAIR principles and open science",
+"Database management": "RDM, FAIR principles and open science",
+"Master data management": "RDM, FAIR principles and open science",
+"Business intelligence": "RDM, FAIR principles and open science",
+"RDM service model": "RDM, FAIR principles and open science",
+
+"Research software": "Research software management",
+"Software management planning": "Research software management",
+"Software version control (e.g., git and GitHub)": "Research software management",
+"Software documentation": "Research software management",
+"Software packaging (R, Python, etc.)": "Research software management",
+"Software citation": "Research software management",
+"FAIR software": "Research software management",
+"Reproducibility": "Research software management",
+"Coding conventions": "Research software management",
+"Literate programming": "Research software management",
+"Scientific workflows and data pipelines": "Research software management",
+"Computer programming": "Research software management",
+"Virtual environments and Containerisation": "Research software management",
+"Continuous integration": "Research software management",
+"Use of generative Artificial Intelligence in writing research software": "Research software management",
+
+"(Certified) Data Repositories": "Data infrastructure",
+"Repository quality standards (e.g. CoreTrustSeal, ISAD(G), OAIS reference model)": "Data infrastructure",
+"TRUST principles for digital repositories": "Data infrastructure",
+"Tool criticism": "Data infrastructure",
+"Data collection tools": "Data infrastructure",
+"Data management services/tools": "Data infrastructure",
+"Data storage (media)": "Data infrastructure",
+"Data transfer tools": "Data infrastructure",
+"Data analysis software/tools": "Data infrastructure",
+"Cloud computing and High-performance computing": "Data infrastructure",
+"Data security and Data classification": "Data infrastructure",
+"Available RDM infrastructure and organisations": "Data infrastructure",
+"European Open Science Cloud solutions": "Data infrastructure",
+
+"European, national and institutional policies on RDM, RSM and open science": "Policy and governance",
+"Funder RDM, RSM and open science requirements": "Policy and governance",
+"Journal policies related to RDM": "Policy and governance",
+"Policy development": "Policy and governance",
+"Policy implementation": "Policy and governance",
+"Policy monitoring": "Policy and governance",
+"Translating policy to organisational strategy": "Policy and governance",
+"Responsible metrics (bibliometrics, altmetrics)": "Policy and governance",
+"Digital sovereignty": "Policy and governance",
+"Data governance": "Policy and governance",
+"Data ownership": "Policy and governance",
+"Key Performance Indicators (KPI) for RDM": "Policy and governance",
+
+"Privacy and data protection (GDPR, UAVG)": "Legal and ethical responsibilities",
+"Sensitive data/Confidentiality": "Legal and ethical responsibilities",
+"Intellectual property rights": "Legal and ethical responsibilities",
+"copyright": "Legal and ethical responsibilities",
+"patents": "Legal and ethical responsibilities",
+"trademarks": "Legal and ethical responsibilities",
+"Research in consortia": "Legal and ethical responsibilities",
+"Data and software licenses ": "Legal and ethical responsibilities",
+"License compatibility": "Legal and ethical responsibilities",
+"Information security": "Legal and ethical responsibilities",
+"Knowledge security": "Legal and ethical responsibilities",
+"European data legislation": "Legal and ethical responsibilities",
+"AI Act": "Legal and ethical responsibilities",
+"Data Governance Act": "Legal and ethical responsibilities",
+"Data Act": "Legal and ethical responsibilities",
+"European Health Data Space": "Legal and ethical responsibilities",
+"(Cyber)security legislation, e.g. NIS2 Directive": "Legal and ethical responsibilities",
+"Trade Secret Protection Act": "Legal and ethical responsibilities",
+"Research ethics and integrity": "Legal and ethical responsibilities",
+"CARE principles": "Legal and ethical responsibilities",
+"Diversity, equity & inclusion": "Legal and ethical responsibilities",
+
+"Needs assessment": "Training and awareness raising",
+"Instructional design": "Training and awareness raising",
+"FAIR-by-Design methodology": "Training and awareness raising",
+"Carpentries methodology": "Training and awareness raising",
+"Training andragogy ": "Training and awareness raising",
+"Didactic methods": "Training and awareness raising",
+"Presentation skills": "Training and awareness raising",
+"Written communication skills": "Training and awareness raising",
+"Open Educational Resources": "Training and awareness raising",
+"Diagnostic, formative and summative assessment": "Training and awareness raising",
+"Course evaluation ": "Training and awareness raising",
+"Student satisfaction": "Training and awareness raising",
+
+"Networking skills": "Transversal skills",
+"Community management": "Transversal skills",
+"Existing RDM networks/ communities": "Transversal skills",
+"Consultancy": "Transversal skills",
+"Advocacy": "Transversal skills",
+"Conflict resolution": "Transversal skills",
+"Negotiating": "Transversal skills",
+"Active listening": "Transversal skills",
+"Stakeholder analysis": "Transversal skills",
+"Stakeholder engagement": "Transversal skills",
+"Organisational development": "Transversal skills",
+"Project management (methodologies)": "Transversal skills",
+"Change management": "Transversal skills",
+"Binding Leadership": "Transversal skills",
+"Facilitation": "Transversal skills",
+"Teamwork": "Transversal skills"
+}

lib/ingestors/auto_parser_mappings/target_audience.json

@@ -0,0 +1,27 @@
+{
+    "post-docs": "researchers",
+    "PhD's candidate": "researchers",
+    "PhD student": "researchers",
+    "principal investigator": "researchers",
+    "professor": "researchers",
+    "scientist": "researchers",
+
+    "library staff": "research support staff",
+    "research librarian": "research support staff",
+    "information specialist": "research support staff",
+    "archivist": "research support staff",
+    "repository manager": "research support staff",
+    "data steward": "research support staff",
+    "data manager": "research support staff",
+    "data professional": "research support staff",
+    "data engineer": "research support staff",
+    "software engineer": "research support staff",
+    "data librarian": "research support staff",
+
+    "bachelor": "students",
+    "master": "students",
+
+    "teacher": "trainers",
+    "coaches": "trainers",
+    "educator": "trainers"
+}

lib/ingestors/auto_parsing.rb

@@ -0,0 +1,23 @@
+module Ingestors
+  module AutoParsing
+    def auto_parse(var, description)
+      @auto_parse_cache ||= {}
+      json_path = File.join(Rails.root, 'lib', 'ingestors', 'auto_parser_mappings', "#{var.to_s}.json")
+      return nil unless File.exist?(json_path)
+
+      mtime = File.mtime(json_path)
+      cached = @auto_parse_cache[var]
+       if cached && cached[:mtime] == mtime
+        mapping = cached[:mapping]
+       else
+        mapping = JSON.parse(File.read(json_path))
+        @auto_parse_cache[var] = { mtime: mtime, mapping: mapping }
+       end
+
+      mapping
+        .select{ |key, val| description&.downcase&.include?(key.to_s.downcase) }
+        &.values
+        &.uniq
+    end
+  end
+end

lib/ingestors/event_ingestion.rb

@@ -1,12 +1,23 @@
 module Ingestors
   module EventIngestion
+    include AutoParsing
+
     def add_event(event)
       if event.is_a?(Hash)
         c = EventsController.new
         c.params = { event: event }
         c.send(:event_params)
         event = OpenStruct.new(c.send(:event_params))
       end
+      TeSS::Config.feature['auto_parse_vars'].each do |var|
+        new_val = auto_parse(var, event.description)
+        next if new_val.blank? 
+
+        current_val = event.send(var) if event.respond_to?(var)
+        if !event.respond_to?(var) || current_val.blank?
+          event.send("#{var}=", new_val)
+        end
+      end
       @events << event unless event.nil?
     end
 
@@ -22,37 +33,6 @@ def convert_location(input)
       input
     end
 
-    def parse_audience(description)
-      audience_mapping = {
-        'post-docs': 'researchers',
-        "PhD's candidate": 'researchers',
-        'PhD student': 'researchers',
-        'principal investigator': 'researchers',
-        'professor': 'researchers',
-        'scientist': 'researchers',
-        'library staff': 'research support staff',
-        'research librarian': 'research support staff',
-        'information specialist': 'research support staff',
-        'archivist': 'research support staff',
-        'repository manager': 'research support staff',
-        'data steward': 'research support staff',
-        'data manager': 'research support staff',
-        'data professional': 'research support staff',
-        'data engineer': 'research support staff',
-        'software engineer': 'research support staff',
-        'data librarian': 'research support staff',
-        'bachelor': 'students',
-        'master': 'students',
-        'teacher': 'trainers',
-        'coaches': 'trainers',
-        'educator': 'trainers',
-      }
-      audience_mapping
-        .select{ |key, val| description.downcase.include?(key.to_s.downcase) }
-        .values
-        .uniq
-    end
-
     def parse_dates(input, timezone = nil)
       Time.use_zone(timezone) do
         # try to split on obvious interval markers

lib/ingestors/material_ingestion.rb

@@ -1,12 +1,23 @@
 module Ingestors
   module MaterialIngestion
+    include AutoParsing
+
     def add_material(material)
       if material.is_a?(Hash)
         c = MaterialsController.new
         c.params = { material: material }
         c.send(:material_params)
         material = OpenStruct.new(c.send(:material_params))
       end
+      TeSS::Config.feature['auto_parse_vars'].each do |var|
+        new_val = auto_parse(var, material.description)
+        next if new_val.blank? 
+
+        current_val = material.send(var) if material.respond_to?(var)
+        if !material.respond_to?(var) || current_val.blank?
+          material.send("#{var}=", new_val)
+        end
+      end
       @materials << material unless material.nil?
     end
   end

lib/ingestors/taxila/dans_ingestor.rb

@@ -64,7 +64,6 @@ def process_dans(url)
 
             event.source = 'DANS'
             event.timezone = 'Amsterdam'
-            event.target_audience = parse_audience(event.description)
 
             add_event(event)
           rescue Exception => e

lib/ingestors/taxila/dtls_ingestor.rb

@@ -72,7 +72,6 @@ def process_dtls(url)
             event.set_default_times
             event.source = 'DTL'
             event.timezone = 'Amsterdam'
-            event.target_audience = parse_audience(event.description)
             add_event(event)
           rescue Exception => e
             @messages << "Extract event fields failed with: #{e.message}"

lib/ingestors/taxila/han_ingestor.rb

@@ -50,7 +50,6 @@ def process_han(_url)
           event.venue = "#{venue_super_css.text} #{venue_sub_css.text}"
           event.source = "HAN"
           event.timezone = 'Amsterdam'
-          event.target_audience = parse_audience(event.description)
 
           add_event(event)
         rescue Exception => e

lib/ingestors/taxila/lcrdm_ingestor.rb

@@ -58,7 +58,6 @@ def process_lcrdm(url)
           event.source = 'LCRDM'
           event.timezone = 'Amsterdam'
           event.set_default_times
-          event.target_audience = parse_audience(event.description)
 
           add_event(event)
         rescue Exception => e

lib/ingestors/taxila/leiden_ingestor.rb

@@ -83,7 +83,6 @@ def process_leiden(url)
             # does TeSS support that?
 
             event.source = 'Universiteit Leiden'
-            event.target_audience = parse_audience(event.description)
 
             add_event(event)
           rescue Exception => e

lib/ingestors/taxila/maastricht_ingestor.rb

@@ -58,7 +58,6 @@ def process_maastricht(url)
             event.timezone = 'Europe/Amsterdam' # how to get this from Icalendar Event object?
 
             event.source = 'Maastricht University'
-            event.target_audience = parse_audience(event.description)
 
             add_event(event)
           rescue Exception => e

lib/ingestors/taxila/nwo_ingestor.rb

@@ -47,7 +47,6 @@ def process_nwo(url)
             event.url = "https://www.nwo.nl#{event_data.css('h3.card__title > a').attribute('href').value}"
 
             event.source = 'NWO'
-            event.target_audience = parse_audience(event.description)
 
             add_event(event)
           rescue Exception => e

lib/ingestors/taxila/odissei_ingestor.rb

@@ -59,7 +59,6 @@ def process_odissei(_url)
           event.source = 'ODISSEI'
           event.timezone = 'Amsterdam'
           event.set_default_times
-          event.target_audience = parse_audience(event.description)
           add_event(event)
         rescue Exception => e
           @messages << "Extract event fields failed with: #{e.message}"

lib/ingestors/taxila/oscd_ingestor.rb

@@ -57,7 +57,6 @@ def process_oscd(url)
             end
             if div&.next_sibling&.next_sibling.nil? || (div&.next_sibling&.next_sibling&.name == 'h1')
               event.set_default_times
-              event.target_audience = parse_audience(event.description)
               add_event(event)
             end
           end

lib/ingestors/taxila/oscm_ingestor.rb

@@ -57,7 +57,6 @@ def process_oscm(url)
             # it's not really needed since dtstart and dtend contain timezone information
             event.source = 'OSCM'
             event.online = true
-            event.target_audience = parse_audience(event.description)
 
             # add event to events array
             add_event(event)

lib/ingestors/taxila/rdnl_ingestor.rb

@@ -57,7 +57,6 @@ def process_rdnl(url)
           event.host_institutions = ['RDNL']
           event.timezone = 'Amsterdam'
           event.set_default_times
-          event.target_audience = parse_audience(event.description)
 
           add_event(event)
         rescue Exception => e

lib/ingestors/taxila/rug_ingestor.rb

@@ -50,7 +50,6 @@ def process_rug(url)
           event.source = 'RUG'
           event.timezone = 'Amsterdam'
           event.set_default_times
-          event.target_audience = parse_audience(event.description)
 
           add_event(event)
         rescue Exception => e

lib/ingestors/taxila/surf_ingestor.rb

@@ -56,7 +56,6 @@ def process_surf(url)
             event.source = 'SURF'
             event.online = true
             event.timezone = 'Amsterdam'
-            event.target_audience = parse_audience(event.description)
 
             # add event to events array
             add_event(event)

lib/ingestors/taxila/tdcc_ingestor.rb

@@ -59,7 +59,6 @@ def process_tdcc(url)
             event.start += 1.year
             event.end += 1.year
           end
-          event.target_audience = parse_audience(event.description)
 
           add_event(event)
         rescue Exception => e