new commit

18_importNumDating.py

@@ -5,82 +5,67 @@ import os # For environment variable loading
 from dotenv import load_dotenv # For environment variable loading
 import pandas as pd # For dataframe handling
 
-# Initialize the database
-print('Initializing the database...')
-engine, metadata, Session = initDb(True, './schemas/')
-if engine == False:
-    print('Database initialization failed.')
-    exit()
+def importNumDating(api, engine):
+    print('Importing num dating...')
 
-# Load the environment variables
-load_dotenv()
+    tableName = "c__5064_num__dat_"
+    bundleId = 'b9cfb95e627e1710cf8d736d4ca5db64' # Dating Information Assignment
 
-# Initialize the WissKI API
-print('Initializing the WissKI API...')
-api_url = os.getenv('API_URL')
-auth = (os.getenv('API_USERNAME'), os.getenv('API_PASSWORD'))
-headers = {"Cache-Control": "no-cache"}
-api = Api(api_url, auth, headers)
-api.pathbuilder = api.get_pathbuilder('default')
+    try:
+        processedRows = pd.read_csv(f'./logs/{tableName}.csv')
+    except FileNotFoundError:
+        processedRows = pd.DataFrame(columns=[ 'id', 'uuid', 'uri'])
 
+    # Load sources table
+    sqlTable = pd.read_sql_table(tableName, con=engine)
 
-tableName = "c__5064_num__dat_"
-bundleId = 'b9cfb95e627e1710cf8d736d4ca5db64' # Dating Information Assignment
-
-try:
-    processedRows = pd.read_csv(f'./logs/processed-{tableName}.csv')
-except FileNotFoundError:
-    processedRows = pd.DataFrame(columns=[ 'docId', 'uuid', 'uri'])
-
-# Load sources table
-sqlTable = pd.read_sql_table(tableName, con=engine)
-
-entityValues = {}
-
-# Create entities
-for index, row in sqlTable.iterrows():
-    # For every row in table...
-    if index < len(processedRows) and sqlTable.loc[index, 'id'] == processedRows.iloc[index, 'docId']:
-        # skip if already processed
-        print(f'Skipping already processed entity {sqlTable.loc[index, 'id']}')
-        continue
-    # Create Entity property dicts
-    entityValues = {}
-    for key, value in row.items():
-        # For every column in row...
-        if (value is None) or (value == ''):
-            #  skip if cell has no value
+    # Create entities
+    for index, row in sqlTable.iterrows():
+        # For every row in table...
+        if index < len(processedRows) and sqlTable.loc[index, 'id'] == processedRows.loc[index, 'id']:
+            # skip if already processed
+            print(f'Skipping already processed entity {sqlTable.loc[index, "id"]}')
             continue
-        # Properties of an entity have to be an array, so...
-        if '&' in str(value):
-            # ...Explode "&"-separated values to array items
-            value = [x.strip() for x in str(value).split('&')]
-        else:
-            # ...Or parse to array
-            value = [value]
-        # Map columns to fields. We use assignments for reification.
-        docId = ''
-        match key:
-            case 'id':
-                docId = value[0]
-            case 'f__uuid':
-                entityValues['f74baaf58e49393cc89d6616ee197901'] = value # UUID
-                uuid = value[0]
-            case 'f__5064_num__dat_':
-                entityValues['f0da3b36d16e16602bb550aff7d36297'] = value # Date
-            case 'f__50bm_bem__datierung':
-                entityValues['fe7870b5a86040d81140bccb01697765'] = value # Note
-            case _:
-                print(f'{key} is not a valid field, skipping.')
+        # Create Entity property dicts
+        entityValues = {}
+        for key, value in row.items():
+            # For every column in row...
+            if (value is None) or (value == ''):
+                #  skip if cell has no value
+                continue
+            # Properties of an entity have to be an array, so...
+            value = str(value).replace('&###{{new_line}}###'.format(), '&')
+            value = str(value).replace('###{{new_line}}###', '&')
+            value = str(value).replace(' & ', '&')
+            if '&' in str(value):
+                # ...Explode "&"-separated values to array items
+                value = [x.strip() for x in str(value).split('&')]
+            else:
+                # ...Or parse to array
+                value = [value]
+            # Map columns to fields. We use assignments for reification.
+            docId = ''
+            match key:
+                case 'id':
+                    docId = value[0]
+                case 'f__uuid':
+                    entityValues['f74baaf58e49393cc89d6616ee197901'] = value # UUID
+                    fUuid = value[0]
+                case 'f__5064_num__dat_':
+                    entityValues['f0da3b36d16e16602bb550aff7d36297'] = value # Date
+                case 'f__50bm_bem__datierung':
+                    entityValues['fe7870b5a86040d81140bccb01697765'] = value # Note
+                case _:
+                    print(f'{key} is not a valid field, skipping.')
 
-    # Create Material
-    entity = Entity(api=api, fields=entityValues, bundle_id=bundleId)
-    api.save(entity)
+        # Create Material
+        entity = Entity(api=api, fields=entityValues, bundle_id=bundleId)
+        api.save(entity)
 
-    print(f'Created entity {index}: {entity.uri} of {len(tableName)}')
+        print(f'Created num dating {index}: {entity.uri} of {len(tableName)}')
 
-    # Write log
-    processedRows = processedRows._append({'docId': docId, 'uuid': fUuid, 'uri': entity.uri}, ignore_index=True)
-    processedRows.to_csv(f'./logs/processed-{tableName}.csv', index=False)
+        # Write log
+        processedRows = processedRows._append({'id': row['id'], 'uuid': fUuid, 'uri': entity.uri}, ignore_index=True)
+        processedRows.to_csv(f'./logs/{tableName}.csv', index=False)
 
-print('finish')
+    print('finish')