diff --git a/common/cordis/transform.py b/common/cordis/transform.py index 03d90ed..e23183f 100644 --- a/common/cordis/transform.py +++ b/common/cordis/transform.py @@ -108,10 +108,13 @@ def parse_organization(ctx: SourceContext, record: SDict): country = "gb" country = get_country_code(country) name = record.pop("name") - vat_id, org_id = record.pop("organisationID"), record.pop("vatNumber") + org_id, vat_id = record.pop("organisationID"), record.pop("vatNumber") if vat_id: if country and not vat_id.lower().startswith(country): - vat_id = f"{country.upper()}{vat_id}" + if country and vat_id.lower().startswith("el"): + pass + else: + vat_id = f"{country.upper()}{vat_id}" proxy.id = ctx.make_slug(vat_id) proxy.add("taxNumber", vat_id) proxy.add("vatCode", vat_id) diff --git a/common/ocds/README.md b/common/ocds/README.md index 1ffcf8f..7c4acde 100644 --- a/common/ocds/README.md +++ b/common/ocds/README.md @@ -218,7 +218,7 @@ ContractAward: eu-ted-award-ocds-213czf-award2 ## See Also -- `util/ocds/eu_ted/` - TED-specific OCDS extraction +- `common/ocds/eu_ted/` - TED-specific OCDS extraction - [OCDS 1.1.5 Schema](https://standard.open-contracting.org/schema/1__1__5/) - [FollowTheMoney Schema](https://followthemoney.tech/explorer/schemata/) - [investigraph Documentation](https://investigraph.dev/) diff --git a/common/ocds/eu_ted/eform_extractors.py b/common/ocds/eu_ted/eform_extractors.py new file mode 100644 index 0000000..48f2ec3 --- /dev/null +++ b/common/ocds/eu_ted/eform_extractors.py @@ -0,0 +1,403 @@ +from lxml import etree + +from common.ocds.eu_ted import model, extractors +from typing import Any, BinaryIO, Generator, List, Optional, Union +from common.ocds.eu_ted.utils import clean +from investigraph.util import make_fingerprint +import uuid +from datetime import datetime + +def extract_eform_address(addr_elem: etree._Element, EFORMS_NSMAP)-> Optional[model.Address]: + if addr_elem is None: + return None + street = addr_elem.findtext( + ".//cbc:StreetName", namespaces=EFORMS_NSMAP + ) + city = addr_elem.findtext( + ".//cbc:CityName", namespaces=EFORMS_NSMAP + ) + postal = addr_elem.findtext( + ".//cbc:PostalZone", namespaces=EFORMS_NSMAP + ) + # check whether matches with ftm codes + country = addr_elem.findtext( + ".//cac:Country/cbc:IdentificationCode", namespaces=EFORMS_NSMAP, + ) + # ?? + nuts = addr_elem.findtext( + ".//cbc:CountrySubentityCode", namespaces=EFORMS_NSMAP + ) + countryCode3 = None + if country is not None: + countryCode3 = extractors.get_alpha3_country_code(country) + + return model.Address( + streetAddress=street, + locality=city, + region=nuts, + postalCode=postal, + countryCode=country, + countryCode3=countryCode3, + + ) + +def extract_org_contact_point( + contact_elem: etree._Element, + EFORMS_NSMAP + ) -> Optional[model.ContactPoint]: + phone = contact_elem.findtext(".//cac:Contact/cbc:Telephone", namespaces=EFORMS_NSMAP) + fax = contact_elem.findtext(".//cac:Contact/cbc:Telefax", namespaces=EFORMS_NSMAP) + email = contact_elem.findtext(".//cac:Contact/cbc:ElectronicMail", namespaces=EFORMS_NSMAP) + name = contact_elem.findtext(".//cac:Contact/cbc:Name", namespaces=EFORMS_NSMAP) + # name = contact_elem.findtext(".//cac:PartyName/cbc:Name", namespaces=EFORMS_NSMAP) + # url = contact_elem.findtext(".//cbc:WebsiteURI", namespaces=EFORMS_NSMAP) + + return model.ContactPoint( + name=name, + telephone=phone, + email=email, + faxNumber=fax, + url=None + # url=url + ) + +def extract_eform_description( + root: etree._Element, + EFORMS_NSMAP +) -> str: + """Extract description from element""" + short_descr = root.findtext(".//cac:ProcurementProject/cbc:Description", namespaces=EFORMS_NSMAP) + note = root.findtext(".//cac:ProcurementProject/cbc:Note", namespaces=EFORMS_NSMAP) + if note is not None: + ret = short_descr + " | note: " + note + else: + ret = short_descr + return clean(ret) + + +def get_lot_result_details( + lot_result_elem: etree._Element, + root: etree._Element, + notice_result: etree._Element, + EFORMS_NSMAP, + orgs: dict[str:model.Organization] +) -> dict: # settled_contract CON-0001 + d = {} + #lot_res_id = lot_result_elem.findtext(".//cbc:ID", namespaces=EFORMS_NSMAP) # RES-0001 - do we really need this? kind of top level + result_code = lot_result_elem.findtext(".//cbc:TenderResultCode", namespaces=EFORMS_NSMAP) # selec-w, clos-nw, open-nw + + lot_tender_id = lot_result_elem.findtext(".//efac:LotTender/cbc:ID", namespaces=EFORMS_NSMAP) # TEN-0001 repeatable + settled_contract_id = lot_result_elem.findtext(".//efac:SettledContract/cbc:ID", namespaces=EFORMS_NSMAP) # CON-0001 repeatable + tender_lot_id = lot_result_elem.findtext(".//efac:TenderLot/cbc:ID", namespaces=EFORMS_NSMAP) # LOT-0001 + d["result-status"] = result_code + d["TEN"] = lot_tender_id + d["CON"] = settled_contract_id + d["LOT"] = tender_lot_id + return d + # lot_tender_elem = result_elem.find(".//efac:TenderLot/cbc:ID", namespaces=EFORMS_NSMAP) + # tendering_party_elem = notice_result.find("", namespaces=EFORMS_NSMAP) + # efac:LotResult (RES-0001) TotalAmount ResultCode -> LotTender TEN-0001 | settledContract CON-0001 | tenderLot LOT-0001 + # efac:LotTender (TEN-0001) -> legalMonetaryTotal | tenderingParty TPA-0001 | tenderLot LOT-0001 | tenderingReference "CON-Submission ID : 300132953" + # efac:SettledContract (CON-0001) -> ContractReference 25.RTI.CP.060 | LotTender TEN-0001 + # efac:TenderingParty (TPA-0001) cbc:Name:SRA4AIR -> Tenderer ORG-0003 ORG-0004 ORG-0005 + +def get_settled_contract_details( + notice_result: etree._Element, + EFORMS_NSMAP, + contract_id: str +) -> dict: + d = {} + contracts = notice_result.findall(".//efac:SettledContract", namespaces=EFORMS_NSMAP) + for contract in contracts: + curr_id = contract.findtext(".//cbc:ID", namespaces=EFORMS_NSMAP) + if curr_id is not None and curr_id == contract_id: + contract_ref = contract.findtext(".//efac:ContractReference/cbc:ID", namespaces=EFORMS_NSMAP) + lot_tender = contract.findtext(".//efac:LotTender/cbc:ID", namespaces=EFORMS_NSMAP) # TEN + d[curr_id] = { + "REF": contract_ref, + "TEN": lot_tender + } + return d +''' +LotTenders + +yes LotResult RES-0001 ResultCode > TEN-0001 , CON-0001, LOT-0001 +yes LotTender TEN-0001 Total Currency TenderReference > TPA-0001 , LOT-0001 +yes SettledContract CON-0001 ContractReference > TEN-0001 +Tenderingparty TPA-0001 > ORG-0001, ORG-0002 GroupLeadIndicator + +non-LotTenders + +ProcurementProjectLot LOT-0001 +''' +def get_tendering_party_details( + notice_result:etree._Element, + root:etree._Element, + EFORMS_NSMAP, + tender_party_id: str +) -> [str]: + arr = [] + tpa_elems = root.findall(".//efac:NoticeResult/efac:TenderingParty", namespaces=EFORMS_NSMAP) + + for elem in tpa_elems: + curr_id = elem.findtext(".//cbc:ID", namespaces=EFORMS_NSMAP) + # print(f"curr supp id: {curr_id} : tender_party_id: {tender_party_id}") + if curr_id is not None and curr_id == tender_party_id: + tenderer_elems = elem.findall(".//efac:Tenderer/cbc:ID", namespaces=EFORMS_NSMAP) + # print(f"item: {item.text}") + for item in tenderer_elems: + # print(f"item: {item.text}") + arr.append(item.text) + # print(f"suppliers: {arr}") + return arr + +def _make_value(value: str, currency: str) -> Optional[model.Value]: + """Extract value with amount and currency""" + # amount_str = extract_text(elem, path) + # currency = extract_attribute(elem, path, "CURRENCY") + + if value is None or currency is None: + return None + + try: + amount = float(value.strip()) + except (ValueError, AttributeError): + return None + + return model.Value(amount=amount, currency=currency) # ISO 4217 string + +def get_lot_tender_details( + notice_result: etree._Element, + root: etree._Element, + EFORMS_NSMAP, + lot_tender_id: str, + opt_amount: str, + opt_curr:str +) -> dict: + d = {} + lot_tenders = root.findall(".//efac:NoticeResult/efac:LotTender", namespaces=EFORMS_NSMAP) + for tender in lot_tenders: + curr_id = tender.findtext(".//cbc:ID", namespaces=EFORMS_NSMAP) + # print(f"curr LotTender id: {curr_id}, lotTenderID: {lot_tender_id}") + if curr_id is not None: + if curr_id == lot_tender_id: + # for elem in tender.iter(): + # print(elem.tag, ":", elem.text) + tender_party = tender.findtext(".//efac:TenderingParty/cbc:ID", namespaces=EFORMS_NSMAP) # TPA-0001 + # print(f"002tender_party: {tender_party}") + amount_elem = tender.find(".//cac:LegalMonetaryTotal/cbc:PayableAmount", namespaces=EFORMS_NSMAP) + tender_value = None + tender_curr = None + if amount_elem is not None: + tender_value = amount_elem.text + tender_curr = amount_elem.get("currencyID") + else: + tender_value = opt_amount + tender_curr = opt_curr + value = _make_value(tender_value, tender_curr) + # cac:LegalMonetaryTotal> Currency + # efac:TenderReference + # efac:TenderLot> LOT-0001 + d["TPA"] = tender_party + d["value"] = value + break + return d + +# extract all the companies/organizations, their contacts and addresses without assigning them a role +#efac:Company/ +def extract_orgs( + root: etree._Element, + EFORMS_NSMAP +) -> dict[str:model.Organization]: + org_elems = root.findall(".//efac:Organizations/efac:Organization/efac:Company", namespaces=EFORMS_NSMAP) + if org_elems is None: + org_elems = root.findall(".//efac:Organizations/efac:Organization/efac:Party", namespaces=EFORMS_NSMAP) + ret = {} + for org_elem in org_elems: + org_id = org_elem.findtext( ".//cac:PartyIdentification/cbc:ID", namespaces=EFORMS_NSMAP) + org_name = org_elem.findtext( ".//cac:PartyName/cbc:Name", namespaces=EFORMS_NSMAP) + org_address = extract_eform_address(org_elem.find(".//cac:PostalAddress", namespaces=EFORMS_NSMAP), EFORMS_NSMAP) + contact_point = None + contact_elem =org_elem.find("./cac:Contact", namespaces=EFORMS_NSMAP) + contact_name = None + if contact_elem is not None: + contact_name = contact_elem.findtext("./cbc:Name", namespaces=EFORMS_NSMAP) + if contact_name is not None: + contact_point = extract_org_contact_point(org_elem, EFORMS_NSMAP) + else: + touch_point_elem = org_elem.find( ".//efac:TouchPoint", namespaces=EFORMS_NSMAP) + if touch_point_elem is not None: + contact_point = extract_org_contact_point(touch_point_elem, EFORMS_NSMAP) + org_email = org_elem.findtext( ".//cac:Contact/cbc:ElectronicMail", namespaces=EFORMS_NSMAP) + org_url = org_elem.findtext(".//cbc:WebsiteURI", namespaces=EFORMS_NSMAP) + org_telephone= org_elem.findtext(".//cac:Contact/cbc:Telephone", namespaces=EFORMS_NSMAP) + org_telefax= org_elem.findtext(".//cac:Contact/cbc:Telefax", namespaces=EFORMS_NSMAP) + org_identifier = model.Identifier( + legalName=org_name, + id = org_elem.findtext(".//cac:PartyLegalEntity/cbc:CompanyID", namespaces=EFORMS_NSMAP), + scheme="National-ID", + ) + # Use fingerprint from name, or generate from identifier if name can't be fingerprinted + org_fingerprint = make_fingerprint(org_name) + if not org_fingerprint: + # Fallback: generate fingerprint from identifier + identifier_str = ( + f"{org_identifier.scheme}:{org_identifier.id}" + if org_identifier.id + else str(org_identifier.legalName) + ) + org_fingerprint = str(uuid.uuid5(uuid.NAMESPACE_URL, identifier_str)) + + org_uuid = uuid.uuid5(uuid.NAMESPACE_URL, org_fingerprint) + org = model.Organization( + id=str(org_uuid), + name=org_name, + identifier=org_identifier, + address= org_address, + contactPoint=contact_point, + details={"url":org_url, "email":org_email, "telephone":org_telephone, "telefax":org_telefax}, + # roles=["buyer"], + ) + ret[org_id] = org + return ret + + +def _extract_award_date_str( + award_date_str: str)->Optional[datetime|None]: + award_date = None + if award_date_str: + try: + temp_value = award_date_str.replace("Z", "+00:00") + award_date = datetime.fromisoformat(temp_value) + # award_date = datetime.strptime(award_date_str, "%Y-%m-%d") + except ValueError: + try: + award_date = datetime.strptime( + award_date_str, "%Y%m%d" + ) + except ValueError: + pass # Leave as None if parsing fails + return award_date + + +''' +Signed Date = Issue Date +Award Date = Award Date +''' +def get_contract_details( + root: etree._Element, + EFORMS_NSMAP, + contract_id: str, + # opt_award_date: str +) -> dict: + d = {} + settled_contract_elems = root.findall(".//efac:NoticeResult/efac:SettledContract", namespaces=EFORMS_NSMAP) + award_date = None + issue_date = None + if settled_contract_elems is not None: + for elem in settled_contract_elems: + curr_id = elem.findtext(".//cbc:ID", namespaces=EFORMS_NSMAP) + if curr_id is not None and curr_id == contract_id: + award_date = _extract_award_date_str(elem.findtext(".//cbc:AwardDate", namespaces=EFORMS_NSMAP)) + issue_date = _extract_award_date_str(elem.findtext(".//cbc:IssueDate", namespaces=EFORMS_NSMAP)) + d["REF"] = elem.findtext(".//efac:ContractReference/cbc::ID", namespaces=EFORMS_NSMAP) + break + + # if award_date is None and opt_award_date is not None: + # award_date = opt_award_date + # if issue_date is None and opt_award_date is not None: + # issue_date = opt_award_date + + d["SIGN"] = issue_date + d["AWARD"] = award_date + + return d + + +def extract_decision_reason( + notice_result: etree._Element, + EFORMS_NSMAP, + lot_res_id: str +)-> str|None: + decision_reason = None + lot_elems = notice_result.findall("./efac:LotResult", namespaces=EFORMS_NSMAP) + curr_elem = None + for elem in lot_elems: + curr_id = elem.findtext("./cbc:ID", namespaces=EFORMS_NSMAP) + if curr_id is not None and curr_id == lot_res_id: + decision_elem = elem.find("./efac:DecisionReason/efbc:DecisionReasonCode", namespaces=EFORMS_NSMAP) + if decision_elem is not None: + if decision_elem.get("listName") is not None: + decision_reason = " | ".join([decision_elem.text, decision_elem.get("listName")]) + else: + decision_reason = decision_elem.text + break + return decision_reason + +#HERE +def extract_contracting_parties(root: etree._Element,EFORMS_NSMAP, organizations:dict[str:model.Organization])-> dict: + d = {} + # find all buyer/contracting party elements + buyer_elems: List = root.findall(".//cac:ContractingParty", + namespaces=EFORMS_NSMAP) + if buyer_elems is not None: + for elem in buyer_elems: + # find buyer id: eg ORG 0001 + curr_id = elem.findtext("./cac:Party/cac:PartyIdentification/cbc:ID", namespaces=EFORMS_NSMAP) + # Type of contracting authority - buyer-legal-type and buyer-contracting-type + buyer_legal_elems = elem.findall("./cac:ContractingPartyType/cbc:PartyTypeCode", namespaces=EFORMS_NSMAP) + # Main activity - authority-activity + buyer_activity = elem.findtext("./cac:ContractingActivity/cbc:ActivityTypeCode", namespaces=EFORMS_NSMAP) + if curr_id is not None: + # find matching organization object + org = organizations.get(curr_id) + if org is not None: + # add role as buyer + org.roles = ["buyer"] + details = [] + buyer_legal_type = None + if buyer_legal_elems is not None: + for element in buyer_legal_elems: + attr = element.get("listName") + if attr is not None and attr == "buyer-legal-type": + buyer_legal_type = element.text + details.append(model.Classification(scheme="TED_CA_TYPE", id=str(buyer_legal_type))) + if buyer_activity is not None: + details.append( + model.Classification(scheme="COFOG", id=str(buyer_activity))) + if details: + org.details = { + "classifications": [c.model_dump() for c in details] + } + d[curr_id] = org + # # find all buyer ids : eg: ORG 0001, ORG 0002 + # ids: List = [e.text for e in root.findall( + # ".//cac:ContractingParty/cac:Party/cac:PartyIdentification/cbc:ID", + # namespaces=EFORMS_NSMAP + # )] + # if ids is not None: #else -> what is the fall back mechanism? + # for item in ids: + # org = None + # org = organizations.get(item) + # if org is not None: + # org.roles=["buyer"] + # d[item] = org + return d + +# def extract_eufunded(lot_elems: etree._Element, EFORMS_NSMAP)->list[str] | None: +# if lot_elems is None: +# return None +# arr: List = [e.findtext( +# "./cac:TenderingTerms/cbc:FundingProgramCode", +# namespaces=EFORMS_NSMAP +# ) for e in lot_elems] +# return arr + +def extract_eufunded(lot_elem: etree._Element, EFORMS_NSMAP)->list[str] | None: + if lot_elem is None: + return None + ret: str = lot_elem.findtext( + "./cac:TenderingTerms/cbc:FundingProgramCode", + namespaces=EFORMS_NSMAP + ) + return ret \ No newline at end of file diff --git a/common/ocds/eu_ted/model.py b/common/ocds/eu_ted/model.py index c3da2e9..16eec54 100644 --- a/common/ocds/eu_ted/model.py +++ b/common/ocds/eu_ted/model.py @@ -133,6 +133,10 @@ class Tender(BaseModel): documents: Optional[List[Any]] = None milestones: Optional[List[Any]] = None amendments: Optional[List[Any]] = None + # extra + cpvCode: Optional[List[str]] = None + cpvName: Optional[List[str]] = None + class Award(BaseModel): @@ -149,6 +153,8 @@ class Award(BaseModel): contractPeriod: Optional[Any] = None documents: Optional[List[Any]] = None amendments: Optional[List[Any]] = None + # extra + decisionReason: Optional[str] = None class Contract(BaseModel): diff --git a/common/ocds/eu_ted/parse.py b/common/ocds/eu_ted/parse.py index eca85f5..e613d85 100644 --- a/common/ocds/eu_ted/parse.py +++ b/common/ocds/eu_ted/parse.py @@ -11,6 +11,8 @@ from pydantic import BaseModel from common.ocds.eu_ted import extractors, model +from common.ocds.eu_ted import eform_extractors as eform +import traceback # Implemented form types IMPLEMENTED_OLD_FORMS = ["F01", "F02", "F03"] @@ -21,6 +23,8 @@ "result", "dir-awa-pre", "cont-modif", + # "can-standard", + # "cn-standard", ] # eForms namespaces @@ -31,8 +35,45 @@ "efbc": "http://data.europa.eu/p27/eforms-ubl-extension-basic-components/1", "efext": "http://data.europa.eu/p27/eforms-ubl-extensions/1", "ext": "urn:oasis:names:specification:ubl:schema:xsd:CommonExtensionComponents-2", + "can": "urn:oasis:names:specification:ubl:schema:xsd:ContractAwardNotice-2", } +def extract_all_cpvs( + tender: model.Tender, + doc_sec_coded: etree._Element, + doc_sec_forms: etree._Element +) -> model.Tender: + + cpv: list = doc_sec_coded.findall( + ".//ORIGINAL_CPV", namespaces=doc_sec_coded.nsmap + ) + if cpv is not None: + cpvCode = [] + cpvName = [] + for item in cpv: + if item is not None: + cpvCode.append(item.get("CODE")) + cpvName.append(item.text) + cpvCode = list(dict.fromkeys(cpvCode)) + cpvName = list(dict.fromkeys(cpvName)) + tender.cpvCode = cpvCode + tender.cpvName = cpvName + + # Check CPV codes in both sections match + other_cpv: list = doc_sec_forms.findall( + ".//CPV_CODE", namespaces=doc_sec_forms.nsmap + ) + other_cpv = list(dict.fromkeys(other_cpv)) + if other_cpv is not None: + tempCPV = [] + for item in other_cpv: + if item is not None: + tempCPV.append(item.get("CODE")) + if set(tender.cpvCode) != set(tempCPV): + print("\033[38;5;208mError: CPV code lists are not same.\033[0m") + + return tender + def ted_notice_to_ocds_releases( doc_id: str, @@ -124,7 +165,7 @@ def ted_notice_to_ocds_releases( cb_pl = doc_sec_forms.find( ".//CONTRACTING_BODY/PROCUREMENT_LAW", namespaces=doc_sec_forms.nsmap ) - if cb_pl is not None and cb_pl: + if cb_pl is not None and len(cb_pl): tender.procurementMethodDetails = cb_pl[0].text # Extract communication details @@ -146,6 +187,9 @@ def ted_notice_to_ocds_releases( tender.description = extractors.extract_description(elem) tender.value = extractors.extract_value(doc_sec_forms, "tender") + # Extract all CPV + tender = extract_all_cpvs(tender, doc_sec_coded, doc_sec_forms) + # Process awards for F03 forms if form_type == "F03": release.awards = [] @@ -407,6 +451,52 @@ def _generate_ocid( return ocid_prefix + str(uuid.uuid5(uuid.NAMESPACE_URL, ocid_suffix)) +def _get_eform_metadata( + root: etree._Element + ): + # get publication date + publication = ( + root.findtext(".//efac:Publication/efbc:PublicationDate", namespaces=EFORMS_NSMAP) + or root.findtext(".//cbc:IssueDate", namespaces=EFORMS_NSMAP) + or root.findtext(".//efac:SettledContract/cbc:IssueDate", namespaces=EFORMS_NSMAP) + or root.findtext(".//cac:ContractAwardNotice/cbc:IssueDate", namespaces=EFORMS_NSMAP) + ) + + # no publication date in the eForm + # if not publication or not publication[0].text: + # return None + # if publication is None: + # return None + # print(f"pub: {publication.tag} {publication.text}") + # date_str = publication[0].text + # print(f"date: {publication} ") + date_str = publication + if date_str is None: + return None + # print(f"date: {date_str} ") + # Remove timezone part if present + if date_str.endswith("Z"): + date_str = date_str[:-1] + elif "+" in date_str: + date_str = date_str.split("+")[0] + elif "-" in date_str and date_str.count("-") > 2: + date_str = date_str.rsplit("-", 1)[0] + + pub_date = datetime.strptime(date_str, "%Y-%m-%d") + + form_type_elem = root.find( + ".//cbc:NoticeTypeCode", + namespaces=EFORMS_NSMAP + ) + + form_type_name = form_type_elem.get("listName") + form_type_text = form_type_elem.text + + return { + "date": pub_date, + "type": form_type_name, + "subtype": form_type_text, + } def parse_eform_notice(root: etree._Element) -> Generator[BaseModel, None, None]: """ @@ -415,16 +505,17 @@ def parse_eform_notice(root: etree._Element) -> Generator[BaseModel, None, None] :param root: Root element of the eForms XML :return: Generator of OCDS releases """ - # Check for eForm type - eform_type_elem = root.find(".//cbc:NoticeTypeCode", EFORMS_NSMAP) - if eform_type_elem is None: - return - eform_type = eform_type_elem.get("listName") + # # Check for eForm type + # eform_type_elem = root.find(".//cbc:NoticeTypeCode", EFORMS_NSMAP) + # if eform_type_elem is None: + # return - # Only process implemented eForms - if eform_type not in IMPLEMENTED_EFORMS: - return + # eform_type = eform_type_elem.get("listName") + + # # Only process implemented eForms + # if eform_type not in IMPLEMENTED_EFORMS: + # return try: # Extract basic notice information @@ -433,46 +524,247 @@ def parse_eform_notice(root: etree._Element) -> Generator[BaseModel, None, None] return doc_id = doc_id_elem.text - date_elem = root.find(".//cbc:IssueDate", EFORMS_NSMAP) - if date_elem is None: + meta = _get_eform_metadata(root) + if meta is None: return - date_str = date_elem.text - # Remove timezone part if present - if date_str.endswith("Z"): - date_str = date_str[:-1] - elif "+" in date_str: - date_str = date_str.split("+")[0] - elif "-" in date_str and date_str.count("-") > 2: - date_str = date_str.rsplit("-", 1)[0] + # if meta.get("type") not in ["competition", "planning", "result", "cont-modif", "dir-awa-pre"]: + # print(f"name:{meta.get("type")} text:{meta.get("subtype")}") - pub_date = datetime.strptime(date_str, "%Y-%m-%d") + pub_date = meta.get("date") + if pub_date is None: + return - # Create basic release - ocid = "ocds-jyvdv7-" + str(uuid.uuid5(uuid.NAMESPACE_URL, doc_id)) + # Create release and tender release = model.Release( - ocid=ocid, - id=str(uuid.uuid5(uuid.NAMESPACE_URL, doc_id)), + ocid="", # Will be set later + id="", # Will be set later date=pub_date, - tag=[model.Tag.tender], + tag=[], # Will be set based on form type initiationType=model.InitiationType.tender, ) + tender = model.Tender(id="") # Will be set to OCID later + release.tender = tender + + form_type = meta.get("type") + if form_type is None: + return + # print(f"form type: {form_type}") + # Initialize parties list for all organizations + parties = [] + + # Set release tags and tender status based on form type + if form_type == "planning": #F01 + notice_type = meta.get("subtype") + if notice_type in ("pin-only", "pin-rtl"): + release.tag = [model.Tag.planning] + tender.status = model.Status.planned + elif form_type == "competition": #F02 + notice_type = meta.get("subtype") + if notice_type == "pin-cfc-standard" or notice_type == "pin-cfc-social": # F01 + release.tag = [model.Tag.planning, model.Tag.tender] + tender.status = model.Status.active + else: # F02 + release.tag = [model.Tag.tender] + tender.status = model.Status.active + elif form_type == "result": #F03 + release.tag = [model.Tag.award, model.Tag.contract] + tender.status = model.Status.complete + + # do we really need uuid component here? + ocid = "ocds-jyvdv7-" + str(uuid.uuid5(uuid.NAMESPACE_URL, doc_id)) + # ocid = "ocds-jyvdv7-" + str(doc_id) + release.ocid = ocid + # Extract title and description title_elem = root.find(".//cac:ProcurementProject/cbc:Name", EFORMS_NSMAP) if title_elem is not None: - release.tender = model.Tender( - id=str(uuid.uuid4()), - title=title_elem.text, - ) + release.tender.title = title_elem.text + + release.tender.id = release.ocid + # release.tender.id = str(uuid.uuid4()) + + # # Extract tender details + # tender.title = extractors.extract_title(elem, doc_sec_trans) + release.tender.description = eform.extract_eform_description(root, EFORMS_NSMAP) + # HERE + # tender.value = extractors.extract_value(doc_sec_forms, "tender") + + cpv_elems: List = root.findall(".//cbc:ItemClassificationCode", EFORMS_NSMAP) + + if cpv_elems is not None and len(cpv_elems) > 0: + arr = [] + for item in cpv_elems: + arr.append(item.text) + + cpvCode = list(dict.fromkeys(arr)) + if len(cpvCode) > 0 and release.tender is not None: + release.tender.cpvCode = cpvCode + # print(cpvCode) + + organizations = eform.extract_orgs(root, EFORMS_NSMAP) + + # extract all eform buyers, first is the primary buyer for OCDS format + buyers = None + buyers_dict= eform.extract_contracting_parties(root,EFORMS_NSMAP, organizations) + if len(buyers_dict) > 0: + first = True + for key,value in buyers_dict.items(): + parties.append(value) + #OCDS supports only one buyer; additional go to parties only + if first: + release.buyer = value + first = False + + release.parties = parties + + # Find ProcurementProjectLot + proc_proj_lot_elems = root.findall("cac:ProcurementProjectLot", namespaces=EFORMS_NSMAP) + + # Process awards for F03 forms - + if form_type == "result": + release.awards = [] + release.contracts = [] + + notice_result = root.find( + ".//efac:NoticeResult", namespaces=EFORMS_NSMAP + ) + if notice_result is None: + print("No .//efac:NoticeResult") + return + + amount_elem = notice_result.find(".//cbc:TotalAmount", namespaces=EFORMS_NSMAP) + amount = None + currency = None + if amount_elem is not None: + amount = amount_elem.text + currency = amount_elem.get("currencyID") + + lot_result_elems=notice_result.findall(".//efac:LotResult", namespaces=EFORMS_NSMAP) + + if len(lot_result_elems) > 0: + # RES-0001 + docs = [] + lot_id = None + award_date = None + signed_date = None + for lot_result_elem in lot_result_elems: + decision_reason = None + lot_res_id = lot_result_elem.findtext(".//cbc:ID", namespaces=EFORMS_NSMAP) # RES-0001 - do we really need this? kind of top level + lot_results = eform.get_lot_result_details(lot_result_elem, root, notice_result, EFORMS_NSMAP, organizations) + contract_details = None + if lot_results.get("CON") is not None: + contract_details = eform.get_contract_details(root, EFORMS_NSMAP, lot_results.get("CON"))#, opt_award_date) + award_date = contract_details.get("AWARD") + signed_date = contract_details.get("SIGN") + result_code = lot_results.get("result-status") + lot_id = lot_results.get("LOT") + if lot_id is not None: + docs.append(lot_id) + value = None + tender_parties = None + tender_party_id = None + if result_code == "selec-w" or result_code == "open-nw": + # settled_con = eform.get_settled_contract_details(notice_result, EFORMS_NSMAP, lot_results.get("CON")) + lot_tender_details= eform.get_lot_tender_details(notice_result, root, EFORMS_NSMAP, lot_results.get("TEN"), amount, currency) + if lot_tender_details is not None: + value = lot_tender_details.get("value") + else: + print(f".//efac:LotTender not available") + + if lot_tender_details is not None: + tender_party_id = lot_tender_details.get("TPA") + # print(f"0001 tendering_party_id: {tender_party_id}") + tender_parties = eform.get_tendering_party_details(notice_result, root, EFORMS_NSMAP, tender_party_id) + else: + print(f"Find tender party without ID?") + else: + decision_reason = eform.extract_decision_reason(notice_result, EFORMS_NSMAP, lot_res_id) + + award_title=None + award_descr=None + award_period = dict() + award_items = [] + + # Find the ProcurementProjectLot + if proc_proj_lot_elems is not None: + for lot in proc_proj_lot_elems: + curr_id = lot.findtext(".//cbc:ID", namespaces=EFORMS_NSMAP) + if curr_id == lot_id: + award_title = lot.findtext(".//cac:ProcurementProject/cbc:Name", namespaces=EFORMS_NSMAP) + award_descr = lot.findtext(".//cac:ProcurementProject/cbc:Description", namespaces=EFORMS_NSMAP) + award_period["startDate"] = eform._extract_award_date_str(lot.findtext(".//cac:ProcurementProject/cac:PlannedPeriod/cbc:StartDate", namespaces=EFORMS_NSMAP)) + award_period["endDate"] = eform._extract_award_date_str(lot.findtext(".//cac:ProcurementProject/cac:PlannedPeriod/cbc:EndDate", namespaces=EFORMS_NSMAP)) + lot_cpv = lot.findtext(".//cac:ProcurementProject/cac:MainCommodityClassification/cbc:ItemClassificationCode", namespaces=EFORMS_NSMAP) + award_items.append({"classification":{"scheme": "CPV"}, "id": lot_cpv}) + eu_funded = eform.extract_eufunded(lot, EFORMS_NSMAP) + if eu_funded is not None: + award_items.append({"classification": { "scheme": "FUNDING", "id": eu_funded}}) + break + + award_id = str(uuid.uuid4()) + award = model.Award( + id=award_id, + title=award_title, + # description=award_descr, + status=model.AwardStatus.active, + date=award_date, + value=value, + items=award_items, + contractPeriod=award_period, + decisionReason=decision_reason, + documents=docs + ) + # For now + if result_code == "selec-w" or result_code == "open-nw": + award.status = model.AwardStatus.active + else: + award.status = model.AwardStatus.unsuccessful + + # Create contract + contract = model.Contract( + id=str(uuid.uuid4()), + awardID=award_id, + title=award_title, + description=award_descr, + status=model.ContractStatus.active, + period=award_period, + value=value, + items=award_items, + dateSigned=signed_date, + documents=docs, + ) + release.contracts.append(contract) + + # Extract suppliers + award.suppliers = [] + if tender_parties is not None: + for item in tender_parties: + supplier = organizations.get(item) + supplier.roles = ["supplier"] + parties.append(supplier) + award.suppliers.append(supplier) + + release.awards.append(award) + else: + # no LotResult section + print(f"no lotResult section") + + # Set parties list + release.parties = parties yield release - except Exception: - # If parsing fails, skip this notice silently + # except Exception: + # # If parsing fails, skip this notice silently + # return + except Exception as e: + # If parsing fails, throw and print the error + print(e) + traceback.print_exc() return - def parse_ted_notice( xml_source: Union[str, BinaryIO], ) -> Generator[BaseModel, None, None]: @@ -515,21 +807,23 @@ def parse_ted_notice( # Extract form type form_type = sec_forms[0].get("FORM") - form_version = sec_forms[0].get("VERSION") + # form_version = sec_forms[0].get("VERSION") if form_type is None: try: form_type = sec_forms[1].get("FORM") - form_version = sec_forms[1].get("VERSION") + # form_version = sec_forms[1].get("VERSION") if form_type is None: return except IndexError: - if form_version is None: - return - else: - return + # if form_version is None: + # return + # else: + # return + return # Normalize form type format + # eg: 1_2014 -> F01_2024 ; 15_2014 -> F15_2014 if form_type[0].isdigit(): if len(form_type) == 1: form_type = f"F0{form_type}" diff --git a/common/ocds/transform.py b/common/ocds/transform.py index 5bdefb4..23e08e6 100644 --- a/common/ocds/transform.py +++ b/common/ocds/transform.py @@ -8,6 +8,7 @@ import logging from banal import is_mapping, ensure_list from investigraph.helpers.addresses import make_address +from common.ocds.eu_ted import model log = logging.getLogger(__name__) @@ -28,6 +29,50 @@ NAME_FIELDS = ["name", "legalName", "entityName", "businessName", "title"] DESCRIPTION_FIELDS = ["description", "summary"] +LEGAL_TYPE = { + # eFORMS + "cga":"PublicBody", # Central government authority + "ra":"PublicBody", # Regional authority + "la":"PublicBody", # Local authority + "body-pl":"PublicBody", # Body governed by public law + "body-pl-cga":"PublicBody", # Body governed by public law, controlled by a central government authority + "body-pl-ra":"PublicBody", # Body governed by public law, controlled by a regional authority + "body-pl-la":"PublicBody", # Body governed by public law, controlled by a local authority + "eu-ins-bod-ag":"PublicBody", # EU institution, body or agency + "grp-p-aut":"PublicBody", # Group of public authorities + + #"pub-undert":"Company", # Public undertaking + #"pub-undert-cga":"Company", # Public undertaking, controlled by a central government authority + #"pub-undert-ra":"Company", # Public undertaking, controlled by a regional authority + #"pub-undert-la":"Company", # Public undertaking, controlled by a local authority + #"spec-rights-entity":"Company", # Entity with special or exclusive rights + + "pub-undert":"PublicBody", # Public undertaking + "pub-undert-cga":"PublicBody", # Public undertaking, controlled by a central government authority + "pub-undert-ra":"PublicBody", # Public undertaking, controlled by a regional authority + "pub-undert-la":"PublicBody", # Public undertaking, controlled by a local authority + "spec-rights-entity":"PublicBody", # Entity with special or exclusive rights + + "def-cont":"Company", # Defence contractor + + "int-org":"LegalEntity", # International organisation + "org-sub":"LegalEntity", # Organisation awarding a contract subsidised by a contracting authority + "org-sub-cga":"LegalEntity", # Organisation awarding a contract subsidised by a central government authority + "org-sub-ra":"LegalEntity", # Organisation awarding a contract subsidised by a regional authority + "org-sub-la":"LegalEntity", # Organisation awarding a contract subsidised by a local authority + + + #OLD format + "BODY_PUBLIC":"PublicBody", + "MINISTRY":"PublicBody", + "REGIONAL_AUTHORITY":"PublicBody", + "EU_INSTITUTION":"PublicBody", + "REGIONAL_AGENCY":"PublicBody", + "NATIONAL_AGENCY":"PublicBody", + + +} + def clean_date(date): """Extract ISO date from datetime string.""" @@ -61,15 +106,33 @@ def get_field_value(data, fields): return None -def determine_org_schema(party_data): +def determine_org_schema(party_data:model.Organization): """Determine appropriate FTM schema based on OCDS party roles.""" + # roles = ensure_list(party_data.get("roles", [])) + # if "buyer" in roles or "procuringEntity" in roles: + # return "PublicBody" + # return "LegalEntity" roles = ensure_list(party_data.get("roles", [])) - if "buyer" in roles or "procuringEntity" in roles: + + # try to retrieve original legal status for buyers + legal_type = None + item = party_data.get("details") + if item is not None: + classifications = item.get("classification", {}) + if classifications is not None: + for classi in classifications: + if classi.get("scheme") == "TED_CA_TYPE": + legal_type = classi.get("id") + break + if "buyer" in roles: + if legal_type is not None and LEGAL_TYPE.get(legal_type): + return LEGAL_TYPE.get(legal_type) + return "PublicBody" + if "procuringEntity" in roles: return "PublicBody" return "LegalEntity" - -def make_organization(ctx, party_data): +def make_organization(ctx, party_data:model.Organization): """Create an Organization entity from OCDS party data. Returns: @@ -166,7 +229,11 @@ def make_organization(ctx, party_data): # Add contact info to organization org.add("email", contact_data.get("email")) org.add("phone", contact_data.get("telephone") or contact_data.get("phone")) + org.add("phone", contact_data.get("faxNumber")) org.add("website", party_data.get("details", {}).get("url")) + org.add("email", party_data.get("details", {}).get("email")) + org.add("phone", party_data.get("details", {}).get("telephone")) + org.add("phone", party_data.get("details", {}).get("telefax")) # Create contact person entity if name is present if contact_name: @@ -196,7 +263,7 @@ def make_organization(ctx, party_data): return org, identification, address_entity, contact, representation -def make_call_for_tenders(ctx, ocid, tender_data, buyer_entity): +def make_call_for_tenders(ctx, ocid, tender_data, buyer_entity, record): """Create CallForTenders entity from OCDS tender data.""" if not is_mapping(tender_data): return None @@ -218,18 +285,33 @@ def make_call_for_tenders(ctx, ocid, tender_data, buyer_entity): # Add submission URL cft.add("sourceUrl", tender_data.get("submissionMethodDetails")) + # Add viewing url from TED + publisher_file = record.get("__fl") + if publisher_file: + cft.add( + "publisherUrl", + "https://ted.europa.eu/en/notice/-/detail/" + + "-".join(publisher_file.split(".xml")[0].split("/")[1].split("_")), + ) + # Add dates from tender period tender_period = tender_data.get("tenderPeriod", {}) if is_mapping(tender_period): cft.add("publicationDate", clean_date(tender_period.get("startDate"))) cft.add("submissionDeadline", clean_date(tender_period.get("endDate"))) - # Add CPV codes from tender items - for item in ensure_list(tender_data.get("items", [])): - if is_mapping(item): - classification = item.get("classification", {}) - if is_mapping(classification) and classification.get("scheme") == "CPV": - cft.add("cpvCode", classification.get("id")) + # # Add CPV codes from tender items - tender class does not have items + # for item in ensure_list(tender_data.get("items", [])): + # if is_mapping(item): + # classification = item.get("classification", {}) + # if is_mapping(classification) and classification.get("scheme") == "CPV": + # cft.add("cpvCode", classification.get("id")) + + # Add CPV for old format + if cft.get("cpvCode") is None or len(cft.get("cpvCode")) < 1: + cpv_code = tender_data.get("cpvCode") + if cpv_code is not None: + cft.add("cpvCode", cpv_code) return cft @@ -257,6 +339,23 @@ def make_contract(ctx, ocid, contract_id, contract_data, call_for_tenders): contract.add("amount", value_data.get("amount")) contract.add("currency", value_data.get("currency")) + # Add viewing url from TED + publisher_file = contract_data.get("__fl") + if publisher_file: + contract.add( + "publisherUrl", + "https://ted.europa.eu/en/notice/-/detail/" + + "-".join(publisher_file.split(".xml")[0].split("/")[1].split("_")), + ) + + for item in ensure_list(contract_data.get("items", [])): + if is_mapping(item): + classification = item.get("classification", {}) + #if is_mapping(classification) and + if classification.get("scheme") == "FUNDING": + if classification.get("id") is not None: + contract.add("summary", classification.get("id")) + return contract @@ -284,6 +383,11 @@ def make_contract_award( award.add("date", clean_date(award_data.get("date") or award_data.get("awardDate"))) award.add("status", award_data.get("status")) + # Add lot number + lot_list = award_data.get("documents") + if lot_list is not None and len(lot_list) == 1: + award.add("lotNumber", lot_list[0]) + # Add value value_data = award_data.get("value", {}) if is_mapping(value_data): @@ -302,6 +406,9 @@ def make_contract_award( classification = item.get("classification", {}) if is_mapping(classification) and classification.get("scheme") == "CPV": award.add("cpvCode", classification.get("id")) + + # Add decision reason (if no winner?) + award.add("decisionReason", award_data.get("decisionReason")) return award @@ -363,7 +470,9 @@ def handle(ctx, record, ix): call_for_tenders = None tender_data = record.get("tender", {}) if is_mapping(tender_data): - call_for_tenders = make_call_for_tenders(ctx, ocid, tender_data, buyer_entity) + call_for_tenders = make_call_for_tenders( + ctx, ocid, tender_data, buyer_entity, record + ) if call_for_tenders: yield call_for_tenders diff --git a/datasets/EU/eu_fts/config.yml b/datasets/EU/eu_fts/config.yml index 979a085..495c4a0 100644 --- a/datasets/EU/eu_fts/config.yml +++ b/datasets/EU/eu_fts/config.yml @@ -68,7 +68,11 @@ extract: "City": beneficiary_city "Postal code": beneficiary_postcode "Beneficiary country": beneficiary_country - "Beneficiary’s contracted amount (EUR)": payment_amount + # "Beneficiary’s contracted amount (EUR)": payment_amount + "Beneficiary’s contracted amount (EUR)": contracted_amount + "Commitment contracted amount (EUR) (A)": commitment_contracted_amount + "Commitment total amount (EUR) (A+B)": commitment_total_amount + "Commitment consumed amount (EUR)": commitment_consumed_amount "Subject of grant or contract": project_identifier "Budget line name": project_name "Programme name": program diff --git a/datasets/EU/eu_fts/transform.py b/datasets/EU/eu_fts/transform.py index 98af9c5..c07414d 100644 --- a/datasets/EU/eu_fts/transform.py +++ b/datasets/EU/eu_fts/transform.py @@ -6,6 +6,7 @@ from investigraph.util import make_data_checksum from investigraph.util import make_fingerprint as fp from investigraph.util import join_text, make_string_id +from followthemoney.types import registry def make_address(ctx: SourceContext, record: Record) -> EntityProxy: @@ -56,10 +57,50 @@ def make_payer(ctx: SourceContext, record: Record) -> EntityProxy | None: return proxy -def make_payment(ctx: SourceContext, record: Record, beneficiary: EntityProxy) -> EntityProxy: +def round_two_decimals_comma(ctx: SourceContext, amount: str) -> str: + if amount: + try: + number_obj = registry.number + float_amount = number_obj.to_number(amount) + rounded = round(float_amount, 2) + amount = f"{rounded:,.2f}" + # amount = str(round(float_amount, 2)) + except Exception as e: + ctx.log.warn(f"Unable to convert to float: {str(e)} : {amount}") + pass + return amount + + +def round_two_decimals(ctx: SourceContext, amount: str) -> str: + if amount: + try: + number_obj = registry.number + float_amount = number_obj.to_number(amount) + amount = str(round(float_amount, 2)) + except Exception as e: + ctx.log.error(f"Unable to convert to float: {str(e)}") + pass + return amount + + +def make_payment( + ctx: SourceContext, record: Record, beneficiary: EntityProxy +) -> EntityProxy: proxy = ctx.make_entity("Payment") + redacted = False proxy.id = ctx.make_id("payment", beneficiary.id, make_data_checksum(record)) - amount = record.pop("payment_amount") + # amount = Beneficiary’s contracted amount -> commitment consumed amount + # amount = record.pop("payment_amount") + amount = record.pop("commitment_consumed_amount") + if amount is None or amount == "*****": + if amount == "*****": + redacted = True + amount = "0" + + amount = round_two_decimals_comma(ctx, amount) + if redacted: + amount = "REDACTED" + proxy.add("amountEur", amount) proxy.add("amount", amount) proxy.add("currency", "EUR") @@ -67,6 +108,39 @@ def make_payment(ctx: SourceContext, record: Record, beneficiary: EntityProxy) - proxy.add("endDate", record["project_endDate"]) proxy.add("date", record["date"]) proxy.add("recordId", record.pop("payment_recordId")) + # added other values in description + contracted_amount = record.get("contracted_amount") or 0 + commitment_amount = record.get("commitment_contracted_amount") or 0 + commitment_total = record.get("commitment_total_amount") or 0 + description = [] + if contracted_amount == "*****": + description.append("Beneficiary’s contracted amount (EUR): REDACTED") + else: + description.append( + f"Beneficiary’s contracted amount (EUR): {round_two_decimals_comma(ctx, contracted_amount)}" + ) + if commitment_amount == "*****": + description.append("Commitment contracted amount (EUR): REDACTED") + else: + description.append( + f"Commitment contracted amount (EUR): {round_two_decimals_comma(ctx, commitment_amount)}" + ) + if commitment_total == "*****": + description.append("Commitment total amount (EUR): REDACTED") + else: + description.append( + f"Commitment total amount (EUR): {round_two_decimals_comma(ctx, commitment_total)}" + ) + + description.append(f"Commitment consumed amount (EUR): {amount}") + + # description = [ + # f'Beneficiary’s contracted amount (EUR): {round_two_decimals_comma(ctx, contracted_amount)}', + # f'Commitment contracted amount (EUR): {round_two_decimals_comma(ctx, commitment_amount)}', + # f'Commitment total amount (EUR): {round_two_decimals_comma(ctx, commitment_total)}', + # f"Commitment consumed amount (EUR): {amount}", + # ] + proxy.add("description", description) return proxy @@ -122,6 +196,9 @@ def make_beneficiary(ctx: SourceContext, record: Record) -> EntityProxy: ident = record.pop("beneficiary_vatCode") proxy.id = ctx.make_slug(ident.upper()) proxy.add("vatCode", ident) + elif record["beneficiary_vatCode"] == "*****": + proxy.id = ctx.make_slug(ident) + proxy.add("vatCode", ident) proxy.add("legalForm", beneficiary_type) proxy.add("name", name) @@ -146,6 +223,44 @@ def handle(ctx: SourceContext, record: Record, ix: int) -> RecordGenerator: payment.add("beneficiary", beneficiary) + if project is not None: + yield make_project_participation(ctx, beneficiary, project, record) + + payment.add("project", project) + payment.add("purpose", project.caption) + yield project + + if payer is not None: + payment.add("payer", payer) + yield payer + + if project is not None: + yield make_project_participation( + ctx, payer, project, record, role="Responsible department" + ) + yield payment + elif record["beneficiary_name"] == "*****": + # with open("redacted.txt", "a") as f: + # f.write(f"{record}\n") + checksum = make_data_checksum(record) + prefix = checksum[:12] + # ctx.log.error("redacted record") + record["beneficiary_name"] = f"REDACTED_{prefix}" + beneficiary = make_beneficiary(ctx, record) + address = make_address(ctx, record) + project = make_project(ctx, record) + payer = make_payer(ctx, record) + payment = make_payment(ctx, record, beneficiary) + + beneficiary.add("country", address.first("country")) + beneficiary.add("address", address.caption) + beneficiary.add("addressEntity", address) + + yield beneficiary + yield address + + payment.add("beneficiary", beneficiary) + if project is not None: yield make_project_participation(ctx, beneficiary, project, record) diff --git a/datasets/EU/eu_ted/extract.py b/datasets/EU/eu_ted/extract.py index 98347cb..c5087c9 100644 --- a/datasets/EU/eu_ted/extract.py +++ b/datasets/EU/eu_ted/extract.py @@ -8,6 +8,7 @@ from investigraph.types import RecordGenerator from common.ocds.eu_ted.parse import parse_ted_notice +import traceback log = logging.getLogger(__name__) @@ -58,6 +59,7 @@ def handle(ctx: C, *args, **kwargs) -> RecordGenerator: ctx.source.info() except (DoesNotExist, FileNotFoundError): ctx.log.warning("Source does not exist", uri=ctx.source.uri) + traceback.print_exc() return processed = 0 @@ -67,12 +69,15 @@ def handle(ctx: C, *args, **kwargs) -> RecordGenerator: with ctx.open() as fh: with tarfile.open(fileobj=fh, mode="r:gz") as tar: for name, xml_fh in _xml_members_from_tar(tar): + # print(f"file: {name}") try: for release in parse_ted_notice(xml_fh): release_dict = release.model_dump( mode="json", exclude_none=True ) release_dict["__source__"] = "eu_ted" + release_dict["__uri"] = ctx.source.uri + release_dict["__fl"] = name yield release_dict total_releases += 1 @@ -94,7 +99,7 @@ def handle(ctx: C, *args, **kwargs) -> RecordGenerator: error=str(e), processed=processed, ) - + traceback.print_exc() ctx.log.info( f"Completed processing {processed} XML files from archive", total_releases=total_releases, diff --git a/datasets/EU/eu_ted/seed.py b/datasets/EU/eu_ted/seed.py index a55feda..1cc5a3e 100644 --- a/datasets/EU/eu_ted/seed.py +++ b/datasets/EU/eu_ted/seed.py @@ -21,6 +21,7 @@ from typing import Generator from investigraph.model import DatasetContext, Source +import traceback # ~22 business days per month (safe upper bound incl. padding for edge cases) ISSUES_PER_MONTH = 23 @@ -100,3 +101,4 @@ def handle(ctx: DatasetContext) -> Generator[Source, None, None]: # Yield in reverse order (newest first) for source in reversed(sources): yield source + traceback.print_exc()