Remove redundant encoding of syllabic segments/syllables from stress and tone enrichment

polyglotdb/corpus/phonological.py

@@ -1,5 +1,3 @@
-import re
-
 from polyglotdb.corpus.lexical import LexicalContext
 from polyglotdb.io.enrichment.features import enrich_features_from_csv, parse_file
 from polyglotdb.io.importer import feature_data_to_csvs, import_feature_csvs
@@ -111,71 +109,60 @@ def enrich_features(self, feature_data, type_data=None):
         self.hierarchy.add_type_properties(self, self.phone_name, type_data.items())
         self.encode_hierarchy()
 
-    def remove_pattern(self, pattern="[0-2]"):
+    def remove_pattern(self, annotation_type: str = None, pattern: str = "[0-2]"):
         """
-        removes a stress or tone pattern from all phones
+        Removes a stress or tone pattern from all phones
 
         Parameters
         ----------
+        annotation_type: str
+            Type of annotation to remove pattern, defaults to phones if not specified
         pattern : str
             the regular expression for the pattern to remove
             Defaults to '[0-2]'
 
         """
-        phone = getattr(self, self.phone_name)
-        if pattern == "":
+        if not annotation_type:
+            annotation_type = self.phone_name
+        if not pattern:
             pattern = "[0-2]"
-        q = self.query_graph(phone)
-        results = q.all()
-        oldphones = []
-        length = 0
-        newphones = []
-        toAdd = {}
-        for item in results:
-            phone = item["label"]
-            if re.search(pattern, phone) is not None:
-                newphone = re.sub(pattern, "", phone)
-                length = len(phone) - len(newphone)
-                oldphones.append(phone)
-                newphones.append(newphone)
-                toAdd.update({"label": newphone})
-        statement = """MATCH (n:{phone_name}{type}:{corpus_name}) WHERE n.label in $oldphones
-        SET n.oldlabel = n.label
-        SET n.label=substring(n.label,0,size(n.label)-{length})"""
+        match_pattern = pattern
+        if not match_pattern.startswith("^"):
+            match_pattern = ".*" + match_pattern
+        if not match_pattern.endswith("$"):
+            match_pattern += ".*"
+        statement = """CYPHER 25
+        MATCH (n:{annotation_type}{type}:{corpus_name}) WHERE n.label =~ $match_regex
+        SET n.old_label = n.label
+        SET n.label=string.regexReplace(n.label, $regex, "")"""
         norm_statement = statement.format(
-            phone_name=self.phone_name,
+            annotation_type=annotation_type,
             type="",
             corpus_name=self.cypher_safe_name,
-            length=length,
         )
         type_statement = statement.format(
-            phone_name=self.phone_name,
+            annotation_type=annotation_type,
             type="_type",
             corpus_name=self.cypher_safe_name,
-            length=length,
         )
-        self.execute_cypher(norm_statement, oldphones=oldphones)
-        self.execute_cypher(type_statement, oldphones=oldphones)
-        self.encode_syllabic_segments(newphones)
-        self.encode_syllables("maxonset")
+        self.execute_cypher(norm_statement, match_regex=match_pattern, regex=pattern)
+        self.execute_cypher(type_statement, match_regex=match_pattern, regex=pattern)
 
-    def reset_to_old_label(self):
+    def reset_to_old_label(self, annotation_type: str = None):
         """
         Reset phones back to their old labels which include stress and tone
+
+        Parameters
+        ----------
+        annotation_type: str
+            Type of annotation to reset, defaults to phones if not specified
         """
-        phones = []
-        getphone = f"""MATCH (n:{self.phone_name}_type:{self.cypher_safe_name})
-        WHERE n.oldlabel IS NOT NULL
-        RETURN n.oldlabel"""
-        results = self.execute_cypher(getphone)
-        for item in results:
-            phones.append(item["n.oldlabel"])
-
-        statement = f"""MATCH (n:{self.phone_name}{{type}}:{self.cypher_safe_name})
-        WHERE n.oldlabel IS NOT NULL SET n.label = n.oldlabel"""
-        norm_statement = statement.format(type="")
+        if not annotation_type:
+            annotation_type = self.phone_name
+
+        statement = f"""MATCH (n:{annotation_type}{{type}}:{self.cypher_safe_name})
+        WHERE n.old_label IS NOT NULL SET n.label = n.old_label"""
+        token_statement = statement.format(type="")
         type_statement = statement.format(type="_type")
-        self.execute_cypher(norm_statement)
+        self.execute_cypher(token_statement)
         self.execute_cypher(type_statement)
-        self.encode_syllabic_segments(phones)
-        self.encode_syllables("maxonset")

polyglotdb/corpus/syllabic.py

@@ -228,7 +228,6 @@ def reset_syllables(self, call_back=None, stop_check=None):
                 self, self.phone_name, ["onset", "coda", "nucleus"]
             )
             self.hierarchy.remove_token_properties(self, self.phone_name, ["syllable_position"])
-            # self.reset_to_old_label()
             self.encode_hierarchy()
         except KeyError:
             pass
@@ -741,7 +740,9 @@ def encode_stress_to_syllables(self, regex=None, clean_phone_label=True):
 
         enrich_dict = self._generate_stress_enrichment(regex, clean_phone_label)
         if clean_phone_label:
-            self.remove_pattern(regex)
+            self.remove_pattern(annotation_type=self.phone_name, pattern=regex)
+            positionless_regex = regex.replace("$", "").replace("^", "")
+            self.remove_pattern(annotation_type="syllable", pattern=positionless_regex)
         self.enrich_syllables(enrich_dict)
         self.encode_hierarchy()
 
@@ -763,7 +764,9 @@ def encode_tone_to_syllables(self, regex=None, clean_phone_label=True):
         enrich_dict = self._generate_tone_enrichment(regex, clean_phone_label)
 
         if clean_phone_label:
-            self.remove_pattern(regex)
+            self.remove_pattern(annotation_type=self.phone_name, pattern=regex)
+            positionless_regex = regex.replace("$", "").replace("^", "")
+            self.remove_pattern(annotation_type="syllable", pattern=positionless_regex)
         self.enrich_syllables(enrich_dict)
         self.encode_hierarchy()
 

polyglotdb/databases/neo4j.conf

@@ -15,6 +15,7 @@
 #dbms.directories.logs=logs
 #dbms.directories.lib=lib
 #dbms.directories.run=run
+db.query.default_language=CYPHER_25
 
 # This setting constrains all `LOAD CSV` import files to be under the `import` directory. Remove or comment it out to
 # allow files to be loaded from anywhere in the filesystem; this introduces possible security problems. See the

tests/test_enrich.py

@@ -1,3 +1,5 @@
+import re
+
 import pytest
 
 from polyglotdb import CorpusContext
@@ -40,6 +42,16 @@ def test_stress_enrichment(stressed_config):
 
         assert c.hierarchy.has_type_property("syllable", "stress")
 
+        q = c.query_lexicon(c.lexicon_phone)
+        q = q.columns(c.lexicon_phone.label.column_name("label"))
+        for r in q.all():
+            assert re.search(r"[0-2]$", r["label"]) is None
+
+        q = c.query_lexicon(c.lexicon_syllable)
+        q = q.columns(c.lexicon_syllable.label.column_name("label"))
+        for r in q.all():
+            assert re.search(r"[0-2]", r["label"]) is None
+
 
 def test_stress_enrichment_no_clean(stressed_config):
     syllabics = "AA0,AA1,AA2,AH0,AH1,AH2,AE0,AE1,AE2,AY0,AY1,AY2,ER0,ER1,ER2,EH0,EH1,EH2,EY1,EY2,IH0,IH1,IH2,IY0,IY1,IY2,UW0,UW1,UW2".split(
@@ -67,6 +79,7 @@ def test_stress_enrichment_no_clean(stressed_config):
         res = q.all()
         for r in res:
             assert r["syllable_stress"] is not None
+            assert re.search(r"[0-2]", r["syllable"]) is not None
 
 
 def test_relativized_enrichment_syllables(acoustic_config):