added some tests for CRF/CRT reading, added symlinks for non-standard CRT files, made some modifications for CRT reading

dodo.py

@@ -498,7 +498,7 @@ def task_test_read_unfccc_crf_for_year():
     return {
         "actions": [
             (read_CRF,),
-            (map_folders, ["extracted_data/UNFCCC"]),
+            # (map_folders, ["extracted_data/UNFCCC"]),
         ],
         "verbosity": 2,
         "setup": ["in_venv"],

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2001-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2001.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2002-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2002.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2003-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2003.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2004-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2004.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2005-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2005.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2006-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2006.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2007-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2007.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2008-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2008.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2009-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2009.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2010-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2010.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2011-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2011.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2012-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2012.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2013-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2013.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2014-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2014.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2015-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2015.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2016-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2016.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2017-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2017.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2018-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2018.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2019-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2019.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2020-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2020.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2021-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2021.xlsx

downloaded_data/UNFCCC/Maldives/BTR1/MDV-CRT-2024-V0.0-2022-20241120-164406_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Maldives/BTR1/2022.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2000-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2000-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2001-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2001-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2002-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2002-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2003-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2003-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2004-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2004-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2005-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2005-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2006-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2006-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2007-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2007-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2008-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2008-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2009-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2009-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2010-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2010-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2011-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2011-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2012-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2012-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2013-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2013-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2014-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2014-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2015-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2015-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2016-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2016-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2017-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2017-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2018-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2018-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2019-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2019-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2020-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2020-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2021-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2021-20241108-153202_started.xlsx

downloaded_data/UNFCCC/Singapore/BTR1/SGP-CRT-2024-V0.3-2022-20241108-153202_started.xlsx

@@ -0,0 +1 @@
+/home/johannes/data/PRIMAP/primap_2.0/datasets/UNFCCC_non-AnnexI_data/downloaded_data/UNFCCC/Singapore/BTR1/CRT (for submission)/SGP-CRT-2024-V0.3-2022-20241108-153202_started.xlsx

src/unfccc_ghg_data/unfccc_crf_reader/unfccc_crf_reader_core.py

@@ -27,6 +27,8 @@ from unfccc_ghg_data.helper import downloaded_data_path_UNFCCC, root_path
 from . import crf_specifications as crf
 from .util import NoCRFFilesError
 
+pd.set_option("future.no_silent_downcasting", True)
+
 
 ### reading functions
 def convert_crf_table_to_pm2if(  # noqa: PLR0913
@@ -432,6 +434,7 @@ def read_crf_table_from_file(  # noqa: PLR0912, PLR0915
             "NULL",
             "NaN",
             "",
+            " ",
         ],
         keep_default_na=False,
     )
@@ -443,7 +446,7 @@ def read_crf_table_from_file(  # noqa: PLR0912, PLR0915
         last_row_nan = False
 
     cols_to_drop = []
-    # remove empty first column (for Australia tables start with an empty column)
+    # remove empty first column (because CRTables start with an empty column)
     # df_raw = df_raw.dropna(how="all", axis=1)
     if df_raw.iloc[:, 0].isna().all():
         cols_to_drop.append(df_raw.columns.to_numpy()[0])
@@ -510,6 +513,7 @@ def read_crf_table_from_file(  # noqa: PLR0912, PLR0915
     # remove double spaces
     entities = [entity.strip() for entity in entities]
     entities = [re.sub("\\s+", " ", entity) for entity in entities]
+    entities = [re.sub("_x000d_ ", "", entity) for entity in entities]
 
     # replace the old header
     if len(header) > 2:  # noqa: PLR2004
@@ -819,7 +823,7 @@ def get_crf_files(  # noqa: PLR0912, PLR0913
     return unique_files
 
 
-def get_info_from_crf_filename(
+def get_info_from_crf_filename(  # noqa: PLR0912
     filename: str,
 ) -> dict[str, Union[int, str]]:
     """
@@ -861,16 +865,31 @@ def get_info_from_crf_filename(
     else:
         # not enough parts, we probably have a CRT file with different separator
         name_parts = filename.split("-")
-        file_info["party"] = name_parts[0]
-        file_info["submission_year"] = int(name_parts[2])
-        file_info["version"] = name_parts[3]
-        try:
-            file_info["data_year"] = int(name_parts[4])
-        except:  # noqa: E722
-            print(f"Data year string {name_parts[4]} " "could not be converted to int.")
-            file_info["data_year"] = name_parts[4]
-        file_info["date"] = name_parts[5]
-        file_info["extra"] = name_parts[6]
+        if len(name_parts) >= 5:  # noqa: PLR2004
+            if name_parts[1] == "CRT":
+                file_info["party"] = name_parts[0]
+                file_info["submission_year"] = int(name_parts[2])
+                file_info["version"] = name_parts[3]
+                try:
+                    file_info["data_year"] = int(name_parts[4])
+                except:  # noqa: E722
+                    print(
+                        f"Data year string {name_parts[4]} "
+                        "could not be converted to int."
+                    )
+                    file_info["data_year"] = name_parts[4]
+                file_info["date"] = name_parts[5]
+                # treat time code and note as optional
+                if len(name_parts) > 6:  # noqa: PLR2004
+                    file_info["extra"] = name_parts[6]
+                else:
+                    file_info["extra"] = ""
+            else:
+                message = f"File {filename} is not a valid CRF or CRT file."
+                raise ValueError(message)
+        else:
+            message = f"File {filename} is not a valid CRF or CRT file."
+            raise ValueError(message)
 
     return file_info
 
@@ -918,9 +937,12 @@ def filter_filenames(
     filtered_files = []
     for file in files_to_filter:
         if not file.is_dir():
-            file_info = get_info_from_crf_filename(file.name)
-            if check_crf_file_info(file_info, file_filter):
-                filtered_files.append(file)
+            try:
+                file_info = get_info_from_crf_filename(file.name)
+                if check_crf_file_info(file_info, file_filter):
+                    filtered_files.append(file)
+            except ValueError:
+                pass
 
     return filtered_files
 
@@ -1155,8 +1177,13 @@ def get_latest_date_for_country(
 
     if type == "CRT":
         type_folder = "BTR"
+        if country_code == "AUS" and submission_year == 1:
+            date_format = "%d%m%Y"
+        else:
+            date_format = "%Y%m%d"
     else:
         type_folder = type
+        date_format = "%d%m%Y"
     if country_code in folder_mapping:
         file_filter = {}
         file_filter["party"] = country_code
@@ -1174,7 +1201,8 @@ def get_latest_date_for_country(
                     / country_folders
                     / f"{type_folder}{submission_year}",
                     file_filter,
-                )
+                ),
+                date_format=date_format,
             )
         else:
             dates = []
@@ -1186,7 +1214,7 @@ def get_latest_date_for_country(
                 )
                 if folder_submission.exists():
                     dates = dates + get_submission_dates(folder_submission, file_filter)
-            submission_date = find_latest_date(dates)
+            submission_date = find_latest_date(dates, date_format=date_format)
     else:
         raise ValueError(  # noqa: TRY003
             f"No data folder found for country {country_code}. "

src/unfccc_ghg_data/unfccc_crf_reader/unfccc_crf_reader_devel.py

@@ -70,6 +70,7 @@ def read_year_to_test_specs(  # noqa: PLR0912, PLR0915
     if country_code == "None":
         country_code = None
 
+    exceptions = []
     unknown_categories = []
     last_row_info = []
     ds_all = None
@@ -89,19 +90,21 @@ def read_year_to_test_specs(  # noqa: PLR0912, PLR0915
             countries_to_read = all_countries
         else:
             raise ValueError("Type must be CRF or CRT")  # noqa: TRY003
-    for country_code in countries_to_read:
+    for current_country_code in countries_to_read:
         # get country name
-        country_name = get_country_name(country_code)
-        print(f"reading for country: {country_code}")
+        country_name = get_country_name(current_country_code)
+        print(f"reading for country: {current_country_code}")
         # get specification and available tables
         # if we only have a single country check if we might have a country specific
         # specification (currently only Australia, 2023)
-        if country_code is not None:
+        if current_country_code is not None:
             try:
-                crf_spec = getattr(crf, f"{type}{submission_year}_{country_code}")
+                crf_spec = getattr(
+                    crf, f"{type}{submission_year}_{current_country_code}"
+                )
                 print(
                     f"Using country specific specification: "
-                    f"{type}{submission_year}_{country_code}"
+                    f"{type}{submission_year}_{current_country_code}"
                 )
             except Exception:
                 # no country specific specification, check for general specification
@@ -140,41 +143,53 @@ def read_year_to_test_specs(  # noqa: PLR0912, PLR0915
 
         try:
             submission_date = get_latest_date_for_country(
-                country_code, submission_year, type=type
+                current_country_code, submission_year, type=type
             )
         except Exception:
-            print(f"No submissions for country {country_name}, {type}{submission_year}")
+            message = (
+                f"No submissions for country {country_name}, {type}{submission_year}"
+            )
+            print(message)
+            exceptions.append(f"No_sub: {country_name}: {message}")
             submission_date = None
+            pass
 
         if submission_date is not None:
             for table in tables:
-                # read table for given years
-                ds_table, new_unknown_categories, new_last_row_info = read_crf_table(
-                    country_code,
-                    table,
-                    submission_year,
-                    date=submission_date,
-                    data_year=[data_year],
-                    debug=True,
-                    type=type,
-                )
-
-                # collect messages on unknown rows etc
-                unknown_categories = unknown_categories + new_unknown_categories
-                last_row_info = last_row_info + new_last_row_info
+                try:
+                    # read table for given years
+                    (
+                        ds_table,
+                        new_unknown_categories,
+                        new_last_row_info,
+                    ) = read_crf_table(
+                        current_country_code,
+                        table,
+                        submission_year,
+                        date=submission_date,
+                        data_year=[data_year],
+                        debug=True,
+                        type=type,
+                    )
 
-                # convert to PRIMAP2 IF
-                # first drop the orig_cat_name col as it can have multiple values for
-                # one category
-                ds_table = ds_table.drop(columns=["orig_cat_name"])
+                    # collect messages on unknown rows etc
+                    unknown_categories = unknown_categories + new_unknown_categories
+                    last_row_info = last_row_info + new_last_row_info
+
+                    # convert to PRIMAP2 IF
+                    # first drop the orig_cat_name col as it can have multiple values
+                    # for one category
+                    ds_table = ds_table.drop(columns=["orig_cat_name"])
+
+                    # TODO: catch entity conversion errors and make list of error
+                    #  entities
+                    # if we need to map entities pass this info to the conversion
+                    # function
+                    if "entity_mapping" in crf_spec[table]:
+                        entity_mapping = crf_spec[table]["entity_mapping"]
+                    else:
+                        entity_mapping = None
 
-                # TODO: catch entity conversion errors and make list of error entities
-                # if we need to map entities pass this info to the conversion function
-                if "entity_mapping" in crf_spec[table]:
-                    entity_mapping = crf_spec[table]["entity_mapping"]
-                else:
-                    entity_mapping = None
-                try:
                     ds_table_if = convert_crf_table_to_pm2if(
                         ds_table,
                         submission_year,
@@ -225,11 +240,11 @@ def read_year_to_test_specs(  # noqa: PLR0912, PLR0915
                     else:
                         ds_all = ds_all.combine_first(ds_table_pm2)
                 except Exception as e:
-                    print(
-                        f"Error occured when converting table {table} for"
-                        f" {country_name} to PRIMAP2 IF. Exception: {e}"
-                    )
-                    # TODO: error handling and logging
+                    message = f"Error occured when converting table {table} for"
+                    f" {country_name} to PRIMAP2 IF. Exception: {e}"
+                    print(message)
+                    exceptions.append(f"Error: {country_name}: {message}")
+                    pass
 
     # process log messages.
     today = date.today()
@@ -292,6 +307,12 @@ def read_year_to_test_specs(  # noqa: PLR0912, PLR0915
     encoding = {var: compression for var in ds_all.data_vars}
     ds_all.pr.to_netcdf(output_folder / (output_filename + ".nc"), encoding=encoding)
 
+    # write exceptions
+    f_ex = open(output_folder / f"exceptions_{output_filename}.txt", "w")
+    for ex in exceptions:
+        f_ex.write(f"{ex}\n")
+    f_ex.close()
+
     return ds_all
 
 

tests/unit/test_crf_reader.py

@@ -1,7 +1,31 @@
 from unfccc_ghg_data.unfccc_crf_reader.unfccc_crf_reader_core import (
     get_info_from_crf_filename,
+    get_latest_date_for_country,
 )
 
+# def test_get_submission_dates()
+#     filter = {}
+#
+#     folder: Path,
+# file_filter: dict[str, Union[str, int, list]],
+
+
+def test_get_latest_date_for_country():
+    # RUS CRF
+    expected = "22082023"
+    date = get_latest_date_for_country("RUS", 2023, type="CRF")
+    assert date == expected
+
+    # AUS CRT
+    expected = "12042024"
+    date = get_latest_date_for_country("AUS", 1, type="CRT")
+    assert date == expected
+
+    # RUS CRT
+    expected = "20241108"
+    date = get_latest_date_for_country("RUS", 1, type="CRT")
+    assert date == expected
+
 
 def test_get_info_from_crf_filename():
     # crf