diff --git a/requirements.txt b/requirements.txt index a1c9719e..bdb60b9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ requests==2.33.1 sentry-sdk==2.59.0 sqlalchemy-utils==0.42.1 urllib3==2.7.0 +orjson==3.11.9 \ No newline at end of file diff --git a/sir/querying.py b/sir/querying.py index 5c8e35f5..8ad000eb 100644 --- a/sir/querying.py +++ b/sir/querying.py @@ -64,6 +64,7 @@ def iterate_path_values(path, obj): """ if obj is None: + yield None return if '.' in path: diff --git a/sir/schema/__init__.py b/sir/schema/__init__.py index 91af4ea0..3b97bd7a 100644 --- a/sir/schema/__init__.py +++ b/sir/schema/__init__.py @@ -37,30 +37,63 @@ from sqlalchemy.orm.descriptor_props import CompositeProperty -SearchAnnotation = E(modelext.CustomAnnotation, [ - F("id", "id"), - F("entity", ["areas.area.gid", "artists.artist.gid", "events.event.gid", - "instruments.instrument.gid", "labels.label.gid", - "places.place.gid", "recordings.recording.gid", - "releases.release.gid", "release_groups.release_group.gid", - "series.series.gid", "works.work.gid"]), - F("name", ["areas.area.name", "artists.artist.name", "events.event.name", - "instruments.instrument.name", "labels.label.name", - "places.place.name", "recordings.recording.name", - "releases.release.name", "release_groups.release_group.name", - "series.series.name", "works.work.name"]), - F("text", "text"), - F("type", ["areas.__tablename__", "artists.__tablename__", - "events.__tablename__", "instruments.__tablename__", - "labels.__tablename__", "places.__tablename__", - "recordings.__tablename__", "releases.__tablename__", - "release_groups.__tablename__", "series.__tablename__", - "works.__tablename__"], - transformfunc=tfs.annotation_type) -], - 1.5, - convert.convert_annotation, - extraquery=queryext.filter_valid_annotations +SearchAnnotation = E( + modelext.CustomAnnotation, + [ + F("id", "id"), + F( + "entity", + [ + "areas.area.gid", + "artists.artist.gid", + "events.event.gid", + "instruments.instrument.gid", + "labels.label.gid", + "places.place.gid", + "recordings.recording.gid", + "releases.release.gid", + "release_groups.release_group.gid", + "series.series.gid", + "works.work.gid", + ], + ), + F( + "name", + [ + "areas.area.name", + "artists.artist.name", + "events.event.name", + "instruments.instrument.name", + "labels.label.name", + "places.place.name", + "recordings.recording.name", + "releases.release.name", + "release_groups.release_group.name", + "series.series.name", + "works.work.name", + ], + ), + F("text", "text"), + F( + "type", + [ + "areas.__tablename__", + "artists.__tablename__", + "events.__tablename__", + "instruments.__tablename__", + "labels.__tablename__", + "places.__tablename__", + "recordings.__tablename__", + "releases.__tablename__", + "release_groups.__tablename__", + "series.__tablename__", + "works.__tablename__", + ], + transformfunc=tfs.annotation_type, + ), + ], + 1.7, + extraquery=queryext.filter_valid_annotations, ) @@ -80,7 +113,7 @@ F("tag", "tags.tag.name"), F("type", "type.name") ], - 1.5, + 1.7, convert.convert_area, extrapaths=["aliases.type.name", "aliases.type.id", "aliases.sort_name", "aliases.type.gid", @@ -107,65 +140,87 @@ F("mbid", "gid"), F("artist", "name"), F("sortname", "sort_name"), - F("alias", "aliases.name"), - # Does not require a trigger since this will get updated on an alias update - F("primary_alias", "primary_aliases", trigger=False), + F("comment", "comment"), + + F("type", "type.name"), + F("type_gid", "type.gid"), + + F("gender_name", "gender.name", transformfunc=tfs.str_list_to_lowercase), + F("gender_gid", "gender.gid"), + F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), F("end", "end_date", transformfunc=tfs.index_partialdate_to_string), F("ended", "ended", transformfunc=tfs.ended_to_string), + + F("country", "area.iso_3166_1_codes.code"), + F("area", ["area.name", "area.aliases.name"]), + F("area_name", "area.name"), + F("area_gid", "area.gid"), + F("area_begindate", "area.begin_date", transformfunc=tfs.index_partialdate_to_string), + F("area_enddate", "area.end_date", transformfunc=tfs.index_partialdate_to_string), + F("area_ended", "area.ended", transformfunc=tfs.ended_to_string), + F("area_type", "area.type.name"), + F("area_type_gid", "area.type.gid"), + F("beginarea", ["begin_area.name", "begin_area.aliases.name"]), - F("country", "area.iso_3166_1_codes.code"), + F("beginarea_name", "begin_area.name"), + F("beginarea_gid", "begin_area.gid"), + F("beginarea_begindate", "begin_area.begin_date", transformfunc=tfs.index_partialdate_to_string), + F("beginarea_enddate", "begin_area.end_date", transformfunc=tfs.index_partialdate_to_string), + F("beginarea_ended", "begin_area.ended", transformfunc=tfs.ended_to_string), + F("beginarea_type", "begin_area.type.name"), + F("beginarea_type_gid", "begin_area.type.gid"), + F("endarea", ["end_area.name", "end_area.aliases.name"]), - F("ref_count", "artist_credit_names.artist_credit.ref_count", - transformfunc=sum, trigger=False), - F("comment", "comment"), - F("gender", "gender.name"), + F("endarea_name", "end_area.name"), + F("endarea_gid", "end_area.gid"), + F("endarea_begindate", "end_area.begin_date", transformfunc=tfs.index_partialdate_to_string), + F("endarea_enddate", "end_area.end_date", transformfunc=tfs.index_partialdate_to_string), + F("endarea_ended", "end_area.ended", transformfunc=tfs.ended_to_string), + F("endarea_type", "end_area.type.name"), + F("endarea_type_gid", "end_area.type.gid"), + + F("alias", "aliases.name", preserve_og=True), + F("alias_json", + ["aliases.name", "aliases.sort_name", "aliases.locale", + "aliases.primary_for_locale", "aliases.begin_date", "aliases.end_date", + "aliases.type.name", "aliases.type.gid"], + objconverter=tfs.aliases_to_json), + # Does not require a trigger since this will get updated on an alias update + F("primary_alias", "primary_aliases", trigger=False), + F("ipi", "ipis.ipi"), F("isni", "isnis.isni"), - F("tag", "tags.tag.name"), - F("type", "type.name") -], - 1.5, - convert.convert_artist, - extrapaths=["tags.count", - "aliases.type.name", "aliases.type.id", - "aliases.type.gid", "aliases.sort_name", - "aliases.locale", "aliases.primary_for_locale", - "aliases.begin_date", "aliases.end_date", - "begin_area.gid", "area.gid", "end_area.gid", - "area.begin_date", "area.end_date", "area.ended", - "begin_area.begin_date", "begin_area.end_date", - "begin_area.ended", "end_area.begin_date", - "end_area.end_date", "end_area.ended", - "gender.gid", "area.type.gid", "area.type.name", - "begin_area.type.gid", "begin_area.type.name", - "end_area.type.gid", "end_area.type.name", - "type.gid"] -) + F("tag", "tags.tag.name", preserve_og=True), + F("tag_count", "tags.count", preserve_og=True), -SearchCDStub = E(modelext.CustomReleaseRaw, [ - F("id", "id"), - F("title", "title"), - F("artist", "artist"), - F("comment", "comment"), - F("barcode", "barcode"), - F("added", "added", transformfunc=tfs.datetime_to_timestamp), - F("tracks", "discids.track_count"), - F("discid", "discids.discid") + F("ref_count", "artist_credit_names.artist_credit.ref_count", + transformfunc=sum, trigger=False), ], - 1.5, - convert.convert_cdstub + 1.7, ) -SearchEditor = E(models.Editor, [ - F("id", "id"), - F("bio", "bio"), - F("editor", "name") -], - 1.5, - convert.convert_editor +SearchCDStub = E( + modelext.CustomReleaseRaw, + [ + F("id", "id"), + F("title", "title"), + F("artist", "artist"), + F("comment", "comment"), + F("barcode", "barcode"), + F("added", "added", transformfunc=tfs.datetime_to_timestamp), + F("tracks", "discids.track_count"), + F("discid", "discids.discid"), + ], + 1.7, +) + +SearchEditor = E( + models.Editor, + [F("id", "id"), F("bio", "bio"), F("editor", "name")], + 1.7, ) SearchEvent = E(modelext.CustomEvent, [ @@ -173,7 +228,7 @@ F("alias", "aliases.name"), F("aid", "area_links.entity0.gid"), F("area", "area_links.entity0.name"), - F("arid", "artist_links.entity0.gid", transformfunc=tfs.uuid_set_to_str_set), + F("arid", "artist_links.entity0.gid", transformfunc=tfs.uuid_list_to_str_list), F("artist", "artist_links.entity0.name"), F("pid", "place_links.entity1.gid"), F("place", "place_links.entity1.name"), @@ -185,7 +240,7 @@ F("ended", "ended", transformfunc=tfs.ended_to_string), F("end", "end_date", transformfunc=tfs.index_partialdate_to_string) ], - 1.5, + 1.7, convert.convert_event, extrapaths=["aliases.type.name", "aliases.type.id", @@ -219,85 +274,129 @@ "time"] ) -SearchInstrument = E(modelext.CustomInstrument, [ - F("alias", "aliases.name"), - F("comment", "comment"), - F("description", "description"), - F("mbid", "gid"), - F("instrument", "name"), - F("tag", "tags.tag.name"), - F("type", "type.name") -], - 1.5, - convert.convert_instrument, - extrapaths=["aliases.type.name", "aliases.type.id", - "aliases.sort_name", "aliases.type.gid", - "aliases.locale", "aliases.primary_for_locale", - "aliases.begin_date", "aliases.end_date", "tags.count", "type.gid"] +SearchInstrument = E( + modelext.CustomInstrument, + [ + F("alias", "aliases.name"), + F("comment", "comment"), + F("description", "description"), + F("mbid", "gid"), + F("instrument", "name"), + F("tag", "tags.tag.name", preserve_og=True), + F("tag_count", "tags.count", preserve_og=True), + F("type", "type.name"), + F("type_gid", "type.gid"), + F( + "alias_json", + [ + "aliases.type.name", + "aliases.type.id", + "aliases.sort_name", + "aliases.type.gid", + "aliases.locale", + "aliases.primary_for_locale", + "aliases.begin_date", + "aliases.end_date", + ], + objconverter=tfs.aliases_to_json, + ), + ], + 1.7, ) -SearchLabel = E(modelext.CustomLabel, [ - F("mbid", "gid"), - F("label", "name"), - F("alias", "aliases.name"), - F("area", ["area.name", "area.aliases.name"]), - F("country", "area.iso_3166_1_codes.code"), - F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), - F("end", "end_date", transformfunc=tfs.index_partialdate_to_string), - F("ended", "ended", transformfunc=tfs.ended_to_string), - - F("code", "label_code"), - F("comment", "comment"), - F("release_count", "release_count", transformfunc=tfs.integer_sum, trigger=False), - F("sortname", "aliases.sort_name"), - F("ipi", "ipis.ipi"), - F("isni", "isnis.isni"), - F("tag", "tags.tag.name"), - F("type", "type.name") -], - 1.5, - convert.convert_label, - extrapaths=["aliases.type.name", "aliases.type.id", - "aliases.type.gid", "aliases.sort_name", - "aliases.locale", "aliases.primary_for_locale", - "aliases.begin_date", "aliases.end_date", - "area.gid", "area.type.name", "area.type.gid", - "area.begin_date", "area.end_date", "area.ended", - "tags.count", "type.gid" - ] +SearchLabel = E( + modelext.CustomLabel, + [ + F("mbid", "gid"), + F("label", "name"), + F("sortname", "aliases.sort_name"), + F("type", "type.name"), + F("type_gid", "type.gid"), + F("area", ["area.name", "area.aliases.name"]), + F("area_name", "area.name"), + F("area_gid", "area.gid"), + F("area_begindate", "area.begin_date", transformfunc=tfs.index_partialdate_to_string), + F("area_enddate", "area.end_date", transformfunc=tfs.index_partialdate_to_string), + F("area_ended", "area.ended", transformfunc=tfs.ended_to_string), + F("area_type_gid", "area.type.gid"), + F("area_type", "area.type.name"), + F("country", "area.iso_3166_1_codes.code"), + F("code", "label_code"), + F("alias", "aliases.name"), + F( + "alias_json", + [ + "aliases.type.name", + "aliases.type.id", + "aliases.type.gid", + "aliases.sort_name", + "aliases.locale", + "aliases.primary_for_locale", + "aliases.begin_date", + "aliases.end_date", + ], + objconverter=tfs.aliases_to_json, + ), + F("ipi", "ipis.ipi"), + F("isni", "isnis.isni"), + F("comment", "comment"), + F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), + F("end", "end_date", transformfunc=tfs.index_partialdate_to_string), + F("ended", "ended", transformfunc=tfs.ended_to_string), + F("tag", "tags.tag.name", preserve_og=True), + F("tag_count", "tags.count", preserve_og=True), + F("release_count", "release_count", transformfunc=tfs.integer_sum, trigger=False), + ], + 1.7, ) -SearchPlace = E(modelext.CustomPlace, [ - F("mbid", "gid"), - F("address", "address"), - F("alias", "aliases.name"), - F("area", ["area.name", "area.aliases.name"]), - F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), - F("comment", "comment"), - F("end", "end_date", transformfunc=tfs.index_partialdate_to_string), - F("ended", "ended", transformfunc=tfs.ended_to_string), - F("lat", "coordinates", transformfunc=tfs.lat), - F("long", "coordinates", transformfunc=tfs.long), - F("place", "name"), - F("type", "type.name") -], - 1.5, - convert.convert_place, - extrapaths=["aliases.type.name", "aliases.type.id", - "aliases.type.gid", "aliases.sort_name", - "aliases.locale", "aliases.primary_for_locale", - "aliases.begin_date", "aliases.end_date", - "area.gid", "area.type.gid", "area.type.name", - "area.begin_date", "area.end_date", "area.ended", - "type.gid"] +SearchPlace = E( + modelext.CustomPlace, + [ + F("mbid", "gid"), + F("place", "name"), + F("address", "address"), + F("comment", "comment"), + F("type", "type.name"), + F("type_gid", "type.gid"), + F("area", ["area.name", "area.aliases.name"]), + F("area_name", "area.name"), + F("area_gid", "area.gid"), + F("area_begindate", "area.begin_date", transformfunc=tfs.index_partialdate_to_string), + F("area_enddate", "area.end_date", transformfunc=tfs.index_partialdate_to_string), + F("area_ended", "area.ended", transformfunc=tfs.ended_to_string), + F("area_type_gid", "area.type.gid"), + F("area_type", "area.type.name"), + F("alias", "aliases.name"), + F( + "alias_json", + [ + "aliases.type.name", + "aliases.type.id", + "aliases.type.gid", + "aliases.sort_name", + "aliases.locale", + "aliases.primary_for_locale", + "aliases.begin_date", + "aliases.end_date", + ], + objconverter=tfs.aliases_to_json, + ), + F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), + F("end", "end_date", transformfunc=tfs.index_partialdate_to_string), + F("ended", "ended", transformfunc=tfs.ended_to_string), + F("lat", "coordinates", transformfunc=tfs.lat), + F("long", "coordinates", transformfunc=tfs.long), + ], + 1.7, ) SearchRecording = E(modelext.CustomRecording, [ F("alias", "aliases.name"), - F("arid", "artist_credit.artists.artist.gid", transformfunc=tfs.uuid_set_to_str_set), + F("arid", "artist_credit.artists.artist.gid", transformfunc=tfs.uuid_list_to_str_list), F("artist", "artist_credit.name"), F("artistname", "artist_credit.artists.artist.name"), F("comment", "comment"), @@ -331,7 +430,7 @@ transformfunc=sum), F("video", "video", transformfunc=tfs.boolean) ], - 1.5, + 1.7, convert.convert_recording, extrapaths=["artist_credit.artists.artist.aliases.begin_date", "artist_credit.artists.artist.aliases.end_date", @@ -384,7 +483,7 @@ F("mbid", "gid"), F("release", "name"), F("alias", "aliases.name"), - F("arid", "artist_credit.artists.artist.gid", transformfunc=tfs.uuid_set_to_str_set), + F("arid", "artist_credit.artists.artist.gid", transformfunc=tfs.uuid_list_to_str_list), F("artist", "artist_credit.name"), F("artistname", "artist_credit.artists.artist.name"), F("asin", "asin.amazon_asin"), @@ -410,10 +509,10 @@ F("status", "status.name"), F("tracks", "mediums.track_count", transformfunc=sum), - F("tracksmedium", "mediums.track_count"), + F("tracksmedium", "mediums.track_count", preserve_og=True), F("tag", "tags.tag.name") ], - 1.5, + 1.7, convert.convert_release, extrapaths=["artist_credit.artists.join_phrase", "artist_credit.artists.artist.aliases.begin_date", @@ -451,7 +550,7 @@ F("mbid", "gid"), F("releasegroup", "name"), F("alias", "aliases.name"), - F("arid", "artist_credit.artists.artist.gid", transformfunc=tfs.uuid_set_to_str_set), + F("arid", "artist_credit.artists.artist.gid", transformfunc=tfs.uuid_list_to_str_list), F("artist", "artist_credit.name"), F("artistname", "artist_credit.artists.artist.name"), F("creditname", "artist_credit.artists.name"), @@ -466,7 +565,7 @@ F("primarytype", "type.name"), F("secondarytype", "secondary_types.secondary_type.name") ], - 1.5, + 1.7, convert.convert_release_group, extrapaths=["artist_credit.artists.join_phrase", "artist_credit.artists.artist.aliases.begin_date", @@ -489,30 +588,40 @@ ) -SearchSeries = E(modelext.CustomSeries, [ - F("mbid", "gid"), - F("alias", "aliases.name"), - F("comment", "comment"), - F("series", "name"), - F("tag", "tags.tag.name"), - F("type", "type.name") -], - 1.5, - convert.convert_series, - extrapaths=["tags.count", - "aliases.type.name", "aliases.type.id", - "aliases.type.gid", "aliases.sort_name", - "aliases.locale", "aliases.primary_for_locale", - "aliases.begin_date", "aliases.end_date", "type.gid"] +SearchSeries = E( + modelext.CustomSeries, + [ + F("mbid", "gid"), + F("series", "name"), + F("type", "type.name"), + F("type_gid", "type.gid"), + F("alias", "aliases.name"), + F( + "alias_json", + [ + "aliases.type.name", + "aliases.type.id", + "aliases.type.gid", + "aliases.sort_name", + "aliases.locale", + "aliases.primary_for_locale", + "aliases.begin_date", + "aliases.end_date", + ], + objconverter=tfs.aliases_to_json, + ), + F("comment", "comment"), + F("tag", "tags.tag.name", preserve_og=True), + F("tag_count", "tags.count", preserve_og=True), + ], + 1.7, ) -SearchTag = E(models.Tag, [ - F("id", "id"), - F("tag", "name") -], - 1.5, - convert.convert_standalone_tag +SearchTag = E( + models.Tag, + [F("id", "id"), F("tag", "name")], + 1.7, ) @@ -527,7 +636,7 @@ "release_links.__tablename__"], transformfunc=tfs.url_type), ], - 1.5, + 1.7, convert.convert_url, extrapaths=["artist_links.entity0.gid", "artist_links.entity0.name", @@ -552,7 +661,7 @@ F("mbid", "gid"), F("work", "name"), F("alias", "aliases.name"), - F("arid", "artist_links.entity0.gid", transformfunc=tfs.uuid_set_to_str_set), + F("arid", "artist_links.entity0.gid", transformfunc=tfs.uuid_list_to_str_list), F("artist", "artist_links.entity0.name"), F("comment", "comment"), F("iswc", "iswcs.iswc"), @@ -563,7 +672,7 @@ F("tag", "tags.tag.name"), F("type", "type.name") ], - 1.5, + 1.7, convert.convert_work, extrapaths=["aliases.type.name", "aliases.type.id", "aliases.type.gid", diff --git a/sir/schema/searchentities.py b/sir/schema/searchentities.py index 01455b4b..2f4f11aa 100644 --- a/sir/schema/searchentities.py +++ b/sir/schema/searchentities.py @@ -117,7 +117,8 @@ class SearchField(object): 3. `gid` column from the `Area` class (model) """ - def __init__(self, name, paths, transformfunc=None, trigger=True): + def __init__(self, name, paths, transformfunc=None, trigger=True, preserve_og=False, + objconverter=None): """ :param str name: The name of the field. :param [str] paths: A dot-delimited path (or a list of them) along which @@ -129,6 +130,11 @@ def __init__(self, name, paths, transformfunc=None, trigger=True): :param method trigger: Whether changes to the path should trigger an update in the Solr core for the entity with this field. Defaults to `True`. + :param method preserve_og: Flag to preserve original values and order. + Defaults to `False`. + :param method objconverter: An optional function that builds this + field's value directly from the entity + object. """ self.name = name if not isinstance(paths, list): @@ -136,6 +142,8 @@ def __init__(self, name, paths, transformfunc=None, trigger=True): self.paths = paths self.transformfunc = transformfunc self.trigger = trigger + self.preserve_og = preserve_og + self.objconverter = objconverter class SearchEntity(object): @@ -261,33 +269,40 @@ def query_result_to_dict(self, obj): data = {} for field in self.fields: fieldname = field.name - tempvals = set() + + if field.objconverter is not None: + value = field.objconverter(obj) + if value: + data[fieldname] = value + continue + + tempvals = list() for path in field.paths: for value in iterate_path_values(path, obj): if value is not None: if isinstance(value, list): - tempvals.update(set(value)) + tempvals.extend(value) else: - tempvals.add(value) + tempvals.append(value) + else: + if field.preserve_og is True: + tempvals.append("") if field.transformfunc is not None: tempvals = field.transformfunc(tempvals) - if (isinstance(tempvals, set) or isinstance(tempvals, list)) and len(tempvals) == 1: - tempvals = tempvals.pop() - if tempvals is not None and tempvals: - if isinstance(tempvals, UUID): - new_tempvals = str(tempvals) - elif isinstance(tempvals, set) or isinstance(tempvals, list): - new_tempvals = list() - for tempval in tempvals: - if isinstance(tempval, UUID): - tempval = str(tempval) - new_tempvals.append(tempval) - else: - new_tempvals = tempvals - data[fieldname] = new_tempvals - if (config.CFG.getboolean("sir", "wscompat") and self.compatconverter is - not None): - data["_store"] = str(tostring(self.compatconverter(obj).to_etree(), encoding='us-ascii'), encoding='us-ascii') + if (isinstance(tempvals, (set, list))): + if field.preserve_og is False: + tempvals = list(set(tempvals)) + if len(tempvals) == 1: + tempvals = tempvals.pop() + + if tempvals is not None and (tempvals or isinstance(tempvals, int)): + if isinstance(tempvals, UUID): + tempvals = str(tempvals) + elif isinstance(tempvals, list): + for idx in range(len(tempvals)): + if isinstance(tempvals[idx], UUID): + tempvals[idx] = str(tempvals[idx]) + data[fieldname] = tempvals return data diff --git a/sir/schema/transformfuncs.py b/sir/schema/transformfuncs.py index e74bd89f..e065778e 100644 --- a/sir/schema/transformfuncs.py +++ b/sir/schema/transformfuncs.py @@ -1,5 +1,6 @@ # Copyright (c) 2014, 2015 Wieland Hoffmann # License: MIT, see LICENSE for details +import orjson from datetime import datetime from sir.wscompat.convert import partialdate_to_string @@ -16,7 +17,7 @@ "release_annotation": "release", "release_group_annotation": "release-group", "series_annotation": "series", - "work_annotation": "work" + "work_annotation": "work", } URL_LINK_TABLE_TO_ENTITYTYPE = { @@ -31,7 +32,7 @@ "l_release_group_url": "release-group", "l_series_url": "series", "l_url_work": "work", - "l_url_url": "url" + "l_url_url": "url", } @@ -41,7 +42,7 @@ def fill_none(values): # in the DB as an empty string, so doing this allows us to search for releases with # `[none]` type barcode via the syntax `barcode:none` if "" in values: - return values.add('none') + return values.append("none") return values @@ -71,6 +72,36 @@ def index_partialdate_to_string(dates): return None +def index_partialdatelist_to_string(date_list): + for idx in range(len(date_list)): + date_list[idx] = partialdate_to_string(date_list[idx]) + return date_list + + +def aliases_to_json(artist): + aliases = [] + for alias in artist.aliases: + data = {"name": alias.name} + if alias.sort_name: + data["sort_name"] = alias.sort_name + if alias.locale: + data["locale"] = alias.locale + if alias.primary_for_locale: + data["primary"] = "primary" + begin_date = partialdate_to_string(alias.begin_date) + if begin_date: + data["begin_date"] = begin_date + end_date = partialdate_to_string(alias.end_date) + if end_date: + data["end_date"] = end_date + if alias.type is not None: + data["type"] = alias.type.name + data["type_id"] = str(alias.type.gid) + + aliases.append(orjson.dumps(data).decode("utf-8")) + return aliases + + def qdur(durations): if len(durations): return durations.pop() // 2000 @@ -109,10 +140,14 @@ def url_type(values): return types -def uuid_set_to_str_set(values): - return {str(x) for x in values} +def uuid_list_to_str_list(values): + return [str(x) for x in values] def datetime_to_timestamp(values): if values: return int(values.pop().timestamp()) + + +def str_list_to_lowercase(values: list[str]) -> list: + return [string.lower() for string in values] diff --git a/test/sql/release.sql b/test/sql/release.sql index 7bb7ad97..85eec28e 100644 --- a/test/sql/release.sql +++ b/test/sql/release.sql @@ -45,6 +45,8 @@ INSERT INTO release (id, gid, name, artist_credit, release_group) VALUES (3, '25 ; INSERT INTO medium (id, gid, track_count, release, position) VALUES (1, 'c517968f-afd0-48e6-ab4b-dfdae888ad9d', 1, 3, 1); +INSERT INTO medium (id, gid, track_count, release, position) VALUES (2, 'e517968f-afd0-48e6-ab4b-dfdae888ad9d', 3, 3, 2); + INSERT INTO recording (id, artist_credit, name, gid) VALUES (1, 2, 'Track on recording', 'b43eb990-ff5b-11de-8a39-0800200c9a66'); INSERT INTO track (id, gid, name, artist_credit, medium, position, number, recording) diff --git a/test/test_indexing_real_data.py b/test/test_indexing_real_data.py index 437cc6c2..520969b2 100644 --- a/test/test_indexing_real_data.py +++ b/test/test_indexing_real_data.py @@ -461,17 +461,17 @@ def test_index_release(self): 'release': u'Blonde on Blonde', 'quality': -1}, { - '_store': 'Various ReleaseVarious ArtistsVarious ArtistsVarious ArtistsVarious Release3', - 'tracks': 3, 'artist': u'Various Artists', + '_store': 'Various ReleaseVarious ArtistsVarious ArtistsVarious ArtistsVarious Release6', + 'tracks': 6, 'artist': u'Various Artists', 'creditname': u'Various Artists', 'artistname': u'Various Artists', 'arid': '7a906020-72db-11de-8a39-0800200c9a66', - 'tracksmedium': 3, + 'tracksmedium': [3,3], 'mbid': '25b6fe30-ff5b-11de-8a39-0800200c9a66', - 'mediumid': 'c517968f-afd0-48e6-ab4b-dfdae888ad9d', + 'mediumid': ['e517968f-afd0-48e6-ab4b-dfdae888ad9d', 'c517968f-afd0-48e6-ab4b-dfdae888ad9d'], 'rgid': '25b6fe30-ff5b-11de-8a39-0800200c9a66', 'release': u'Various Release', - 'mediums': 1, + 'mediums': 2, 'quality': -1 } ] diff --git a/test/test_searchentities.py b/test/test_searchentities.py index 58fa8352..ee562700 100644 --- a/test/test_searchentities.py +++ b/test/test_searchentities.py @@ -17,10 +17,9 @@ def setUp(self): self.entity = E(models.B, [ F("id", "id"), F("c_bar", "c.bar"), - F("c_bar_trans", "c.bar", transformfunc=lambda v: - v.union(set(["yay"]))) + F("c_bar_trans", "c.bar", transformfunc=lambda v: v + ["yay"]) ], - 1.1 + 1.1 ) self.expected = { "id": 1,