import uuid # For UUID creation from initDb import initDb # For database initialization from wisski.api import Api, Pathbuilder, Entity # For WissKI API import os # For environment variable loading from dotenv import load_dotenv # For environment variable loading import pandas as pd # For dataframe handling def importMarks(api, engine): print('Importing marks...') tableName = 'c__mar' bundleId = 'b2c4e1c984d7758d7c7ec719110f7125' try: processedRows = pd.read_csv(f'./logs/{tableName}.csv') except FileNotFoundError: processedRows = pd.DataFrame(columns=['id', 'markId', 'uuid', 'uri']) # Load mark table sqlTable = pd.read_sql_table(tableName, con=engine) print(f'Processing {len(sqlTable)} marks...') # Create mark for index, row in sqlTable.iterrows(): # For every row in table... if index < len(processedRows) and sqlTable.loc[index, 'id'] == processedRows.loc[index, 'id']: # skip if already processed print(f'Skipping already processed mark {sqlTable.loc[index, "id"]}') continue # Create Entity property dicts markValues = {} creationValues = {} digitisationProcessValues = {'f32274ec0032b8778ba69d20108590cc': [str(uuid.uuid4())]} dimensionValues = {} featureValues = {} featureDimensionValues = {} imageValues = {} imageAssignmentValues = {'f067784f5b1ff850672124a2b05360de': [str(uuid.uuid4())]} for key, value in row.items(): # For every column in row... if (value is None) or (value == ''): # skip if cell has no value continue # Properties of an entity have to be an array, so... value = str(value).replace('&###{{new_line}}###'.format(), '&') value = str(value).replace('###{{new_line}}###', '&') value = str(value).replace(' & ', '&') if '&' in str(value): # ...Explode "&"-separated values to array items value = [x.strip() for x in str(value).split('&')] else: # ...Or parse to array value = [value] # Map columns to fields. We use assignments for reification # for nested semantics, because we need to be efficient. match key: case 'id': continue case 'f__uuid': markValues['fb40b199b4032e55acc152f994e93b45'] = value # UUID case 'f__3002_pub_kue_nr_': markValues['f6f0572ebec9c98e164d0e9aa0650c2e'] = value # Artist Number case 'f__6700_mar_dok_nr_': markValues['fe577970c02f173170ff3848a36b3b79'] = value # Mark Document Number case 'f__6770_rosenb_nr_': markValues['f6fc4b5726c97bad8b03ede860491649'] = value # Rosenberg Number case 'f__9990_kommentar': markValues['f01e527e707ff36bf966baa01c163378'] = value # Comment case 'f__68an_abdruck_nr_': markValues['f8324ea3c9ee378f1e19035e092aadb9'] = value # Print Number case 'f__68nk_besonderheiten': markValues['fa21e323a8a7a99ce3489e1f7753ac5f'] = value # Special Features case 'f__8470_aufnahmenr_': markValues['f67031e2a2b81ad9f318dc5b11d5a6af'] = value # Recording number case 'f__684b_breite_marke': # We map dimensions to Dimension entity. dimensionValues.setdefault(key, {})['f31e9c7e2de5549daea1790a74615288'] = ['width'] # Type dimensionValues[key]['f3f805d270890837a6493e7e60a96487'] = value # Dimension dimensionValues[key]['f802fd7bf45be523a9b188411a591420'] = [str(uuid.uuid4())] # UUID case 'f__684h_hoehe_marke': dimensionValues.setdefault(key, {})['f31e9c7e2de5549daea1790a74615288'] = ['hight'] # Type dimensionValues[key]['f3f805d270890837a6493e7e60a96487'] = value # Dimension dimensionValues[key]['f802fd7bf45be523a9b188411a591420'] = [str(uuid.uuid4())] # UUID case 'f__68na_bz_breite_hoehe': dimensionValues.setdefault(key, {})['f31e9c7e2de5549daea1790a74615288'] = ['width_x_hight'] # Type dimensionValues[key]['f3f805d270890837a6493e7e60a96487'] = value # Dimension dimensionValues[key]['f802fd7bf45be523a9b188411a591420'] = [str(uuid.uuid4())] # UUID case 'f__6840_rahmenform': # We map features to Feature entity. featureValues.setdefault(key, {})['fdfb3c4f670aa1260924cecd09ca4bbb'] = ['frame_shape'] # Type featureValues[key]['fbccee184fa531d58b3b46eb8ac4626f'] = value # Feature featureValues[key]['f299e2a145b508e376f2bf2e44cbe219'] = [str(uuid.uuid4())] # UUID case 'f__684d_darst__marke': # We map features to Feature entity. featureValues.setdefault(key, {})['fdfb3c4f670aa1260924cecd09ca4bbb'] = ['design'] # Type featureValues[key]['fbccee184fa531d58b3b46eb8ac4626f'] = value # Feature featureValues[key]['f299e2a145b508e376f2bf2e44cbe219'] = [str(uuid.uuid4())] # UUID case 'f__684l_text_marke': # We map features to Feature entity. featureValues.setdefault(key, {})['fdfb3c4f670aa1260924cecd09ca4bbb'] = ['text'] # Type featureValues[key]['fbccee184fa531d58b3b46eb8ac4626f'] = value # Feature featureValues[key]['f299e2a145b508e376f2bf2e44cbe219'] = [str(uuid.uuid4())] # UUID case 'f__68nb_randanschluss': # We map features to Feature entity. featureValues.setdefault(key, {})['fdfb3c4f670aa1260924cecd09ca4bbb'] = ['edge_connection'] # Type featureValues[key]['fbccee184fa531d58b3b46eb8ac4626f'] = value # Feature featureValues[key]['f299e2a145b508e376f2bf2e44cbe219'] = [str(uuid.uuid4())] # UUID case 'f__68nc_form_haste': # We map features to Feature entity. featureValues.setdefault(key, {})['fdfb3c4f670aa1260924cecd09ca4bbb'] = ['haste_mould'] # Type featureValues[key]['fbccee184fa531d58b3b46eb8ac4626f'] = value # Feature featureValues[key]['f299e2a145b508e376f2bf2e44cbe219'] = [str(uuid.uuid4())] # UUID case 'f__68nd_form_schraegstr_': # We map features to Feature entity. featureValues.setdefault(key, {})['fdfb3c4f670aa1260924cecd09ca4bbb'] = ['slash_form_shape'] # Type featureValues[key]['fbccee184fa531d58b3b46eb8ac4626f'] = value # Feature featureValues[key]['f299e2a145b508e376f2bf2e44cbe219'] = [str(uuid.uuid4())] # UUID case 'f__68ne_haste_schraegstr_': # We map features to Feature entity. featureValues.setdefault(key, {})['fdfb3c4f670aa1260924cecd09ca4bbb'] = ['transition_haste_slash'] # Type featureValues[key]['fbccee184fa531d58b3b46eb8ac4626f'] = value # Feature featureValues[key]['f299e2a145b508e376f2bf2e44cbe219'] = [str(uuid.uuid4())] # UUID case 'f__68nf_n_knick': # We map features to Feature entity. featureValues.setdefault(key, {})['fdfb3c4f670aa1260924cecd09ca4bbb'] = ['transition_haste_slash_kink'] # Type featureValues[key]['fbccee184fa531d58b3b46eb8ac4626f'] = value # Feature featureValues[key]['f299e2a145b508e376f2bf2e44cbe219'] = [str(uuid.uuid4())] # UUID case 'f__68ng_ueberg__serifen': # We map features to Feature entity. featureValues.setdefault(key, {})['fdfb3c4f670aa1260924cecd09ca4bbb'] = [ 'transition_serif_haste'] # Type featureValues[key]['fbccee184fa531d58b3b46eb8ac4626f'] = value # Feature featureValues[key]['f299e2a145b508e376f2bf2e44cbe219'] = [str(uuid.uuid4())] # UUID case 'f__68nh_dicke_ser__max_': # We map (features) dimensions to Dimension entity. featureDimensionValues.setdefault(key, {})['f31e9c7e2de5549daea1790a74615288'] = ['maximum_thickness'] # Type featureDimensionValues[key]['f3f805d270890837a6493e7e60a96487'] = value # Dimension featureDimensionValues[key]['f802fd7bf45be523a9b188411a591420'] = [str(uuid.uuid4())] # UUID case 'f__68ni_dicke_ser__min': # We map features to Feature entity. featureDimensionValues.setdefault(key, {})['f31e9c7e2de5549daea1790a74615288'] = [ 'minimum_thickness'] # Type featureDimensionValues[key]['f3f805d270890837a6493e7e60a96487'] = value # Dimension featureDimensionValues[key]['f802fd7bf45be523a9b188411a591420'] = [str(uuid.uuid4())] # UUID case 'f__68nj_breite_serife': # We map features to Feature entity. featureDimensionValues.setdefault(key, {})['f31e9c7e2de5549daea1790a74615288'] = [ 'width'] # Type featureDimensionValues[key]['f3f805d270890837a6493e7e60a96487'] = value # Dimension featureDimensionValues[key]['f802fd7bf45be523a9b188411a591420'] = [str(uuid.uuid4())] # UUID case 'f__8540_repro_nr_': # We map images to Image entity for item in value: if item is not None: # Replace dir paths in name item = item.replace('Objekte\\', 'objects/') item = item.replace('Objekte3\\', 'objects/') item = item.replace('Objekte4\\', 'objects/') item = item.replace('objekte4\\', 'objects/') item = item.replace('Objekte5\\', 'objects/') item = item.replace('objekte5\\', 'objects/') item = item.replace('Marken\\', 'marks/') item = item.replace('Marken/', 'marks/') item = item.replace('MArken\\', 'marks/') item = item.replace('Goldschmiede/', 'goldsmiths/') item = item.replace('Goldschmiede\\', 'goldsmiths/') item = item.replace('Epitaphien/', 'epitaphies/') item = item.replace('Epitaphien\\', 'epitaphies/') imageValues.setdefault(item, {})['feb10344eaa7a5f414d1e8392853eba9'] = [item] # Reproduction Number (Image) imageValues[item]['fc8d57e55f203c75c2f8a1ae79378ac7'] = ['public://artifact_images/' + item + '.jpg'] # File imageValues[item]['f11beac4b638016479e6f3fbc7e55d1a'] = [str(uuid.uuid4())] # UUID case 'f__9900_datum_erfassung': digitisationProcessValues['f1f5dd22371e5c1de41e0fb099e0e862'] = value # Recording date case 'f__99ae_datum_aenderung': digitisationProcessValues['f8976c6a9e5d91fe9caba8a57c27f204'] = value # Change date case 'f__efbm_bem_erfassung': digitisationProcessValues['f78a6310d13c717b82ddba814ac59024'] = value # Recording note case 'f__ptxt_plug_in_text': markValues['ffb8b04e8d57929a596fc32d6a84d07d'] = value # Plugin text case _: print(f'{key} is not a valid field, skipping.') # Create Dimension entities and add their UUIDs to a list # because we link Mark and Dimension over the UUID dimension = [] for key, value in dimensionValues.items(): if value: dimensionItem = Entity(api=api, fields=value, bundle_id='b73258adf62f35bd1be3fa2863fab558') api.save(dimensionItem) dimension.append(value['f802fd7bf45be523a9b188411a591420'][0]) # Create (feature) Dimension entities and add their UUIDs to a list # because we link Feature and its Dimension over the UUID featureDimension = [] for key, value in featureDimensionValues.items(): if value: featureDimensionItem = Entity(api=api, fields=value, bundle_id='b73258adf62f35bd1be3fa2863fab558') # Dimension Bundle api.save(featureDimensionItem) featureDimension.append(value['f802fd7bf45be523a9b188411a591420'][0]) # Dimension UUID # Add the serif feature t the feature list if featureDimension: featureValues.setdefault('serif', {})['fdfb3c4f670aa1260924cecd09ca4bbb'] = ['serif'] # Feature Type featureValues['serif']['f0f825f5d3a6f0e2d67eee311b94cd6f'] = featureDimension # Dimension UUIDs featureValues['serif']['f299e2a145b508e376f2bf2e44cbe219'] = [str(uuid.uuid4())] # UUID # Create Dimension entities and add their UUIDs to a list # because we link Mark and Dimension over the UUID feature = [] for key, value in featureValues.items(): if value: featureItem = Entity(api=api, fields=value, bundle_id='b393e1c3db202fbb7a8b54e65eb38227') # Feature Bundle api.save(featureItem) feature.append(value['f299e2a145b508e376f2bf2e44cbe219'][0]) # Feature UUID # Create Image entities and add their UUIDs to a list # because we link Image Assignment and Image over the UUID imageList = [] for key, value in imageValues.items(): if value: imageItem = Entity(api=api, fields=value, bundle_id='b8c6c4b478ead1c80e175ad0f98dafe3') api.save(imageItem) imageList.append(value['f11beac4b638016479e6f3fbc7e55d1a'][0]) # Create Image Assignment entities and add their UUIDs to a list # because we link Artifact and Image Assignment over the UUID if imageList: imageAssignmentValues['f70afb79b45472fee3d02f011caa4b36'] = imageList # List of Image UUIDs imageAssignment = Entity(api=api, fields=imageAssignmentValues, bundle_id='b88e5d94fb2a83d62df99cf64d6c010c') api.save(imageAssignment) # Create Digitisation Process if digitisationProcessValues: digitisationProcess = Entity(api=api, fields=digitisationProcessValues, bundle_id='b22e6c47ccb3ab8a974b37279e1bc33b') api.save(digitisationProcess) # Add the field values for reference if dimension: markValues['f05807c9d81cd39b814f83de0175d66a'] = dimension # Dimension if feature: markValues['f3ce49288bc03e9d799f20ea277429db'] = feature # Feature if imageAssignmentValues['f067784f5b1ff850672124a2b05360de'][0]: markValues['f73e27498813a922032b18b3f3ab8d10'] = [imageAssignmentValues['f067784f5b1ff850672124a2b05360de'][0]] # Image Assignment if digitisationProcessValues['f32274ec0032b8778ba69d20108590cc'][0]: markValues['f3baf98f752fc9638de175985183119a'] = [digitisationProcessValues['f32274ec0032b8778ba69d20108590cc'][0]] # Digitisation Process # Create Mark mark = Entity(api=api, fields=markValues, bundle_id=bundleId) api.save(mark) print(f'Created mark number {index}: {mark.uri} of {len(sqlTable)}') # Write log processedRows = processedRows._append({'id': row['id'], 'markId': markValues['fe577970c02f173170ff3848a36b3b79'][0], 'uuid': markValues['fb40b199b4032e55acc152f994e93b45'][0], 'uri': mark.uri}, ignore_index=True) processedRows.to_csv(f'./logs/{tableName}.csv', index=False) print('finished importing marks')