Merge branch 'develop' into feature/type_graphics

config/database_versions.py

@@ -1,5 +1,6 @@
 # Used for automatic database upgrades and database version checks
 DATABASE_VERSIONS = [
+    '7.15.0',
     '7.14.0',
     '7.13.0',
     '7.11.0',

config/default.py

@@ -79,7 +79,9 @@
     "entityID", "typeID", "typeIDWithSubs", "relationToID"]
 SET_CATEGORIES: list[str] = ["valueTypeID"]
 VALID_CATEGORIES: list[str] = [
-    *STR_CATEGORIES, *INT_CATEGORIES, *SET_CATEGORIES]
+    *STR_CATEGORIES,
+    *INT_CATEGORIES,
+    *SET_CATEGORIES]
 COMPARE_OPERATORS: list[str] = [
     'equal', 'notEqual', 'greaterThan', 'lesserThan', 'greaterThanEqual',
     'lesserThanEqual', 'like']
@@ -91,6 +93,10 @@
     'base_url': None,
     'thumbnail_url': 'https://arche-thumbnails.acdh.oeaw.ac.at/'}
 
+# Used to connect to password protected Vocabs systems
+VOCABS_USER = ''
+VOCABS_PW = ''
+
 # Table options
 TABLE_ROWS = {10: '10', 25: '25', 50: '50', 100: '100'}
 

install/3_data_web.sql

@@ -15,7 +15,7 @@ INSERT INTO web.user (username, password, active, email, group_id) VALUES (
   (SELECT id FROM web.group WHERE name = 'admin'));
 
 INSERT INTO web.settings (name, value) VALUES
-  ('database_version', '7.14.0'),
+  ('database_version', '7.15.0'),
   ('api_public', ''),
   ('default_language', 'en'),
   ('table_rows', '25'),
@@ -45,4 +45,7 @@ INSERT INTO web.settings (name, value) VALUES
   ('profile_image_width', '200'),
   ('random_password_length', '16'),
   ('reset_confirm_hours', '24'),
-  ('site_name', 'OpenAtlas');
+  ('site_name', 'OpenAtlas'),
+  ('vocabs_base_url', 'https://vocabs.acdh.oeaw.ac.at/'),
+  ('vocabs_endpoint', 'rest/v1/'),
+  ('vocabs_user', '');

install/upgrade/7.15.0.sql

@@ -0,0 +1,15 @@
+-- Upgrade 7.14.x to 7.15.0
+-- Be sure to backup the database and read the upgrade notes before executing.
+
+BEGIN;
+
+-- Raise database version
+UPDATE web.settings SET value = '7.15.0' WHERE name = 'database_version';
+
+-- (#1991) Import controlled vocabularies via API
+INSERT INTO web.settings (name, value) VALUES
+    ('vocabs_base_url', 'https://vocabs.acdh.oeaw.ac.at/'),
+    ('vocabs_endpoint', 'rest/v1/'),
+    ('vocabs_user', '');
+
+END;

install/upgrade/upgrade.md

@@ -19,8 +19,8 @@ then run the database upgrade script, then restart Apache:
     sudo python3 install/upgrade/database_upgrade.py
     sudo service apache2 restart
 
-### 7.14.x to 7.14.2
-A code base update (e.g. with git pull) and a webserver restart is sufficient.
+### 7.14.x to 7.15.0
+7.15.0.sql is needed but will be taken care of by the database upgrade script.
 
 ### 7.13.x to 7.14.0
 7.14.0.sql is needed but will be taken care of by the database upgrade script.

openatlas/__init__.py

@@ -27,7 +27,7 @@
 from openatlas.views import (
     admin, ajax, arche, tools, changelog, entity, entity_index, error, export,
     file, hierarchy, index, imports, link, login, model, note, overlay,
-    profile, search, sql, type as type_, user)
+    profile, search, sql, type as type_, user, vocabs)
 
 @babel.localeselector
 def get_locale() -> str:

openatlas/api/arche/function.py → openatlas/api/import_scripts/arche.py

@@ -1,17 +1,19 @@
-from typing import Any, Optional
+from typing import Any
 
 import rdflib
 import requests
-from flask import g, flash
-from requests import Response, HTTPError
+from flask import flash, g
+from requests import HTTPError, Response
 from werkzeug.exceptions import abort
 
 from openatlas import app
+from openatlas.api.import_scripts.util import (
+    get_exact_match, get_or_create_type, get_reference_system)
+from openatlas.database.gis import Gis as Db_gis
+from openatlas.database.reference_system import ReferenceSystem as Db
 from openatlas.models.entity import Entity
 from openatlas.models.imports import is_float
-from openatlas.database.gis import Gis as Db_gis
 from openatlas.models.reference_system import ReferenceSystem
-from openatlas.database.reference_system import ReferenceSystem as Db
 from openatlas.models.type import Type
 
 
@@ -63,12 +65,8 @@ def get_metadata(data: dict[str, Any]) -> dict[str, Any]:
 def import_arche_data() -> int:
     count = 0
     person_types = get_or_create_person_types()
-    arche_ref = \
-        [i for i in g.reference_systems.values() if i.name == 'ARCHE'][0]
-    exact_match_id = \
-        get_or_create_type(
-            Type.get_hierarchy('External reference match'),
-            'exact match').id
+    arche_ref = get_reference_system('ARCHE')
+    exact_match_id = get_exact_match().id
     for entries in fetch_arche_data().values():
         for item in entries.values():
             name = item['name']
@@ -138,23 +136,6 @@ def get_linked_image(data: list[dict[str, Any]]) -> str:
             if str(image['mime'][0]) == 'image/jpeg'][0]
 
 
-def get_or_create_type(hierarchy: Entity, type_name: str) -> Entity:
-    if type_ := get_type_by_name(type_name):
-        if type_.root[0] == hierarchy.id:
-            return type_
-    type_entity = Entity.insert('type', type_name)
-    type_entity.link('P127', hierarchy)
-    return type_entity
-
-
-def get_type_by_name(type_name: str) -> Optional[Type]:
-    type_ = None
-    for type_id in g.types:
-        if g.types[type_id].name == type_name:
-            type_ = g.types[type_id]
-    return type_
-
-
 def get_hierarchy_by_name(name: str) -> Type:
     for type_id in g.types:
         if g.types[type_id].name == name:

openatlas/api/import_scripts/util.py

@@ -0,0 +1,47 @@
+from typing import Any, Optional
+
+import requests
+from flask import g
+
+from openatlas import app
+from openatlas.models.entity import Entity
+from openatlas.models.type import Type
+
+
+def get_or_create_type(hierarchy: Entity, type_name: str) -> Entity:
+    if type_ := get_type_by_name(type_name):
+        if type_.root[0] == hierarchy.id:
+            return type_
+    type_entity = Entity.insert('type', type_name)
+    type_entity.link('P127', hierarchy)
+    return type_entity
+
+
+def get_type_by_name(type_name: str) -> Optional[Type]:
+    type_ = None
+    for type_id in g.types:
+        if g.types[type_id].name == type_name:
+            type_ = g.types[type_id]
+    return type_
+
+
+def get_exact_match() -> Entity:
+    return get_or_create_type(
+        Type.get_hierarchy('External reference match'), 'exact match')
+
+
+def get_reference_system(name: str) -> Entity:
+    return [i for i in g.reference_systems.values() if i.name == name][0]
+
+
+def vocabs_requests(
+        id_: Optional[str] = '',
+        endpoint: Optional[str] = '',
+        parameter: Optional[dict[str, str]] = '') -> dict[str, Any]:
+    req = requests.get(
+        f"{g.settings['vocabs_base_url']}{g.settings['vocabs_endpoint']}{id_}/"
+        f"{endpoint}",
+        params=parameter,
+        timeout=60,
+        auth=(app.config['VOCABS_USER'], app.config['VOCABS_PW']))
+    return req.json()

openatlas/api/import_scripts/vocabs.py

@@ -0,0 +1,128 @@
+from typing import Any, Optional
+
+from flask import g
+
+from openatlas.api.import_scripts.util import get_exact_match, vocabs_requests
+from openatlas.database.reference_system import ReferenceSystem as Db
+from openatlas.models.entity import Entity
+from openatlas.models.reference_system import ReferenceSystem
+from openatlas.models.type import Type
+
+
+def import_vocabs_data(
+        id_: str,
+        form_data: dict[str, Any],
+        details: dict[str, Any]) -> tuple[list, list]:
+    return fetch_top_level(id_, details, form_data)
+
+
+def fetch_top_level(
+        id_: str,
+        details: dict[str, Any],
+        form_data: dict[str, Any]) -> tuple[list, list]:
+    req = vocabs_requests(id_, 'topConcepts', {'lang': form_data['language']})
+    count = []
+    duplicates = []
+    if ref := get_vocabs_reference_system(details):
+        for entry in req['topconcepts']:
+            if entry['uri'] in form_data['top_concepts'] \
+                    and not Type.check_hierarchy_exists(entry['label']):
+                hierarchy = Entity.insert(
+                    'type',
+                    entry['label'],
+                    f'Automatically imported from {details["title"]}')
+                Type.insert_hierarchy(
+                    hierarchy,
+                    'custom', form_data['classes'],
+                    form_data['multiple'])
+                entry['subs'] = import_children(
+                    entry['uri'],
+                    id_,
+                    form_data['language'],
+                    ref,
+                    hierarchy)
+                count.append(entry)
+            if Type.check_hierarchy_exists(entry['label']):
+                duplicates.append(entry)
+    return count, duplicates
+
+
+def import_children(
+        uri: str,
+        id_: str,
+        lang: str,
+        ref: ReferenceSystem,
+        super_: Optional[Entity], ) -> list[dict[str, Any]]:
+    req = vocabs_requests(id_, 'narrower', {'uri': uri, 'lang': lang})
+    exact_match_id = get_exact_match().id
+    children = []
+    child = None
+    for entry in req['narrower']:
+        if not entry['prefLabel']:
+            g.logger.log(
+                'warn',
+                'import',
+                f'{entry["uri"]} has no prefLabel assigned to it')
+            continue
+        name = entry['uri'].rsplit('/', 1)[-1]
+        if super_:
+            child = Entity.insert(
+                'type',
+                entry['prefLabel']  # Switch if bug is solved
+                # get_pref_label(entry['prefLabel'], id_, entry['uri'])
+            )
+            child.link('P127', super_)
+            ref.link('P67', child, name, type_id=exact_match_id)
+        entry['subs'] = import_children(entry['uri'], id_, lang, ref, child)
+        children.append(entry)
+    return children
+
+
+# Skosmos API has a problem, this code will work if bug is closed
+#
+# def get_pref_label(label: str, id_: str, uri: str) -> str:
+#     if not label:
+#         req = vocabs_requests(id_, 'label', {'uri': uri})
+#         label = req['prefLabel']
+#     return label
+
+
+def get_vocabs_reference_system(details: dict[str, Any], ) -> ReferenceSystem:
+    title = details['title']
+    system = None
+    for system_ in g.reference_systems.values():
+        if system_.name == f'{title} vocabulary':
+            system = system_
+    if not system:
+        system = ReferenceSystem.insert_system({
+            'name': f'{title} vocabulary',
+            'description': f'Import of {title} vocabulary (autogenerated)',
+            'website_url': g.settings['vocabs_base_url'],
+            'resolver_url': f"{details['conceptUri'].rsplit('/', 1)[0]}/"})
+        Db.add_classes(system.id, ['type'])
+    return system
+
+
+def get_vocabularies() -> list[dict[str, Any]]:
+    req = vocabs_requests(endpoint='vocabularies', parameter={'lang': 'en'})
+    out = []
+    for voc in req['vocabularies']:
+        out.append(voc | fetch_vocabulary_details(voc['uri']))
+    return out
+
+
+def fetch_vocabulary_details(id_: str) -> dict[str, str]:
+    data = vocabs_requests(id_, parameter={'lang': 'en'})
+    return {
+        'id': data['id'],
+        'title': data['title'],
+        'defaultLanguage': data['defaultLanguage'],
+        'languages': data['languages'],
+        'conceptUri':
+            data['conceptschemes'][0]['uri'] if data['conceptschemes'] else ''}
+
+
+def fetch_top_concept_details(id_: str) -> list[tuple]:
+    req = vocabs_requests(id_, 'topConcepts', parameter={'lang': 'en'})
+    return [
+        (concept['uri'], concept['label']) for concept in req['topconcepts']]

openatlas/api/swagger.json

@@ -838,7 +838,7 @@
         "required" : true,
         "in" : "path",
         "schema" : {
-          "enum" : [ "E6", "E7", "E8", "E9", "E12", "E18", "E20", "E21", "E22", "E31", "E32", "E33", "E41", "E53", "E54", "E55", "E74" ]
+          "enum" : [ "E5", "E7", "E8", "E9", "E12", "E18", "E20", "E21", "E22", "E31", "E32", "E33", "E41", "E53", "E54", "E55", "E65","E74" ]
         },
         "example" : "E18"
       },

openatlas/forms/form.py

@@ -6,7 +6,7 @@
 from flask_babel import lazy_gettext as _
 from flask_wtf import FlaskForm
 from wtforms import HiddenField, SelectMultipleField, StringField, widgets
-from wtforms.validators import InputRequired
+from wtforms.validators import InputRequired, URL
 
 from openatlas import app
 from openatlas.display.table import Table
@@ -104,3 +104,14 @@ class Form(FlaskForm):
     form = Form(obj=type_)
     form.selection.choices = choices
     return form
+
+
+def get_vocabs_form() -> FlaskForm:
+    class Form(FlaskForm):
+        base_url = StringField(
+            _('base URL'),
+            validators=[InputRequired(), URL()])
+        endpoint = StringField(_('endpoint'), validators=[InputRequired()])
+        vocabs_user = StringField(_('user'))
+        save = SubmitField(_('save'))
+    return Form()

openatlas/static/manual/_static/documentation_options.js

@@ -1,6 +1,6 @@
 var DOCUMENTATION_OPTIONS = {
     URL_ROOT: document.getElementById("documentation_options").getAttribute('data-url_root'),
-    VERSION: '7.14.0',
+    VERSION: '7.15.0',
     LANGUAGE: 'en',
     COLLAPSE_INDEX: false,
     BUILDER: 'html',

openatlas/templates/admin/data.html

@@ -23,6 +23,7 @@ <h1 class="mb-1">{{ _('data transfer')|uc_first }}</h1>
     {% if config.ARCHE['id'] %}
        <div class="col-auto">{{ 'ARCHE'|button(url_for('arche_index'))|safe }}</div>
     {% endif %}
+    <div class="col-auto">{{ 'VOCABS'|button(url_for('vocabs_index'))|safe }}</div>
   </div>
 {% endif %}
 {% if 'admin'|is_authorized and g.settings.image_processing %}

openatlas/views/arche.py

@@ -4,7 +4,8 @@
 from werkzeug.wrappers import Response
 
 from openatlas import app
-from openatlas.api.arche.function import fetch_arche_data, import_arche_data
+from openatlas.api.import_scripts.arche import (
+    fetch_arche_data, import_arche_data)
 from openatlas.database.connect import Transaction
 from openatlas.display.tab import Tab
 from openatlas.display.table import Table
@@ -25,7 +26,9 @@ def arche_index() -> str:
             display_info({
                 k: str(v) for k, v in app.config['ARCHE'].items()}),
             buttons=[manual('admin/arche')])},
-        crumbs=['ARCHE'])
+        crumbs=[
+            [_('admin'), f"{url_for('admin_index')}#tab-data"],
+            'ARCHE'])
 
 
 @app.route('/arche/fetch')