refactor country definitions

src/unfccc_ghg_data/helper/__init__.py

@@ -6,7 +6,9 @@ the unfccc_ghg_data package
 """
 
 from .definitions import (
+    AI_countries,
     GWP_factors,
+    all_countries,
     code_path,
     compression,
     custom_country_mapping,
@@ -20,6 +22,7 @@ from .definitions import (
     gas_baskets,
     legacy_data_path,
     log_path,
+    nAI_countries,
     root_path,
 )
 from .functions import (
@@ -57,4 +60,7 @@ __all__ = [
     "get_code_file",
     "compression",
     "make_wide_table",
+    "nAI_countries",
+    "AI_countries",
+    "all_countries",
 ]

src/unfccc_ghg_data/helper/definitions.py

@@ -3,6 +3,8 @@
 import os
 from pathlib import Path
 
+import pandas as pd
+
 
 def get_root_path() -> Path:
     """Get the root_path from an environment variable"""
@@ -27,6 +29,15 @@ legacy_data_path = root_path / "legacy_data"
 dataset_path = root_path / "datasets"
 dataset_path_UNFCCC = dataset_path / "UNFCCC"
 
+nAI_countries = list(
+    pd.read_csv(code_path / "unfccc_di_reader" / "DI_NAI_parties.conf")["code"]
+)
+# AI_countries = list(reader.annex_one_reader.parties["code"])
+AI_countries = list(
+    pd.read_csv(code_path / "unfccc_di_reader" / "DI_AI_parties.conf")["code"]
+)
+
+all_countries = nAI_countries + AI_countries
 
 custom_country_mapping = {
     "EUA": "European Union",

src/unfccc_ghg_data/unfccc_crf_reader/unfccc_crf_reader_prod.py

@@ -10,6 +10,7 @@ import primap2 as pm2
 import xarray as xr
 
 from unfccc_ghg_data.helper import (
+    all_countries,
     code_path,
     custom_country_mapping,
     extracted_data_path_UNFCCC,
@@ -375,8 +376,14 @@ def read_new_crf_for_year(
         list[str]: list with country codes for which the data has been read
 
     """
-    if countries is None:
-        countries = all_crf_countries
+    if type == "CRF":
+        if countries is None:
+            countries = all_crf_countries
+    elif type == "CRT":
+        if countries is None:
+            countries = all_countries
+    else:
+        raise ValueError("Type must be CRF or CRT")  # noqa: TRY003
 
     read_countries = {}
     for country in countries:
@@ -482,6 +489,7 @@ def read_new_crf_for_year_datalad(
                     country_info["name"],
                     submission_year=submission_year,
                     submission_date=country_info["date"],
+                    type=type,
                     verbose=False,
                 )
                 if not data_read:
@@ -657,6 +665,8 @@ def submission_has_been_read(  # noqa: PLR0913
     """
     output_folder = extracted_data_path_UNFCCC / country_name.replace(" ", "_")
     output_filename = f"{country_code}_{type}{submission_year}_{submission_date}"
+
+    #    check if the submission_year is correctly used for CRT
     if output_folder.exists():
         existing_files = output_folder.glob(f"{output_filename}.*")
         existing_suffixes = [file.suffix for file in existing_files]
@@ -667,21 +677,25 @@ def submission_has_been_read(  # noqa: PLR0913
                     f"Data already available for {country_code}, "
                     f"{type}{submission_year}, version {submission_date}."
                 )
-        else:
+        elif existing_suffixes:
             has_been_read = False
             if verbose:
                 print(
                     f"Partial data available for {country_code}, "
                     f"{type}{submission_year}, version {submission_date}. "
                     "Please check if all files have been written after "
-                    "reading."
+                    f"reading. Existing suffixes: {existing_suffixes}"
+                )
+        else:
+            has_been_read = False
+            if verbose:
+                print(
+                    f"No read data available for {country_code}, "
+                    f"{type}{submission_year}, version {submission_date}. "
                 )
     else:
         has_been_read = False
         if verbose:
-            print(
-                f"No read data available for {country_code}, "
-                f"{type}{submission_year}, version {submission_date}. "
-            )
+            print(f"No read data available for {country_code}. ")
 
     return has_been_read

src/unfccc_ghg_data/unfccc_di_reader/unfccc_di_reader_core.py

@@ -13,6 +13,8 @@ import pycountry
 import unfccc_di_api
 import xarray as xr
 
+from unfccc_ghg_data.helper import AI_countries, nAI_countries
+
 from .unfccc_di_reader_config import (
     cat_code_regexp,
     di_query_filters,
@@ -20,7 +22,7 @@ from .unfccc_di_reader_config import (
     di_to_pm2if_template_nai,
 )
 from .unfccc_di_reader_io import save_DI_country_data, save_DI_dataset
-from .util import AI_countries, DI_date_format, nAI_countries
+from .util import DI_date_format
 
 
 def read_UNFCCC_DI_for_country(  # noqa: PLR0913

src/unfccc_ghg_data/unfccc_di_reader/unfccc_di_reader_proc.py

@@ -8,12 +8,12 @@ from typing import Optional, Union
 import primap2 as pm2
 import xarray as xr
 
-from unfccc_ghg_data.helper import gas_baskets, process_data_for_country
+from unfccc_ghg_data.helper import gas_baskets, nAI_countries, process_data_for_country
 
 from .unfccc_di_reader_config import cat_conversion, di_processing_info
 from .unfccc_di_reader_helper import determine_filename, find_latest_DI_data
 from .unfccc_di_reader_io import save_DI_country_data, save_DI_dataset
-from .util import DI_date_format, nAI_countries
+from .util import DI_date_format
 
 
 def process_and_save_UNFCCC_DI_for_country(

src/unfccc_ghg_data/unfccc_di_reader/util.py

@@ -1,19 +1,11 @@
 """
 Country group and error definitions for DI reader
 """
-import pandas as pd
 
-from unfccc_ghg_data.helper import code_path
 
 # reader = unfccc_di_api.UNFCCCApiReader()
 # nAI_countries = list(reader.non_annex_one_reader.parties["code"])
-nAI_countries = list(
-    pd.read_csv(code_path / "unfccc_di_reader" / "DI_NAI_parties.conf")["code"]
-)
-# AI_countries = list(reader.annex_one_reader.parties["code"])
-AI_countries = list(
-    pd.read_csv(code_path / "unfccc_di_reader" / "DI_AI_parties.conf")["code"]
-)
+
 
 DI_date_format = "%Y-%m-%d"
 regex_date = r"([0-9]{4}-[0-9]{2}-[0-9]{2})"