diff --git a/requirements.txt b/requirements.txt index a1c9719e..bdb60b9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ requests==2.33.1 sentry-sdk==2.59.0 sqlalchemy-utils==0.42.1 urllib3==2.7.0 +orjson==3.11.9 \ No newline at end of file diff --git a/sir/querying.py b/sir/querying.py index 5c8e35f5..8ad000eb 100644 --- a/sir/querying.py +++ b/sir/querying.py @@ -64,6 +64,7 @@ def iterate_path_values(path, obj): """ if obj is None: + yield None return if '.' in path: diff --git a/sir/schema/__init__.py b/sir/schema/__init__.py index 91af4ea0..ef104a31 100644 --- a/sir/schema/__init__.py +++ b/sir/schema/__init__.py @@ -24,82 +24,133 @@ # Paths might need to be updated as a part of schema changes in the # MusicBrainz server. -from sir.schema import queryext -from sir.schema import modelext -from sir.schema import transformfuncs as tfs -from sir.schema.searchentities import SearchEntity as E, SearchField as F -from sir.wscompat import convert -from collections import OrderedDict +from collections import OrderedDict, defaultdict + from mbdata import models -from collections import defaultdict from sqlalchemy.orm import class_mapper -from sqlalchemy.orm.properties import ColumnProperty, RelationshipProperty from sqlalchemy.orm.descriptor_props import CompositeProperty +from sqlalchemy.orm.properties import ColumnProperty, RelationshipProperty - -SearchAnnotation = E(modelext.CustomAnnotation, [ - F("id", "id"), - F("entity", ["areas.area.gid", "artists.artist.gid", "events.event.gid", - "instruments.instrument.gid", "labels.label.gid", - "places.place.gid", "recordings.recording.gid", - "releases.release.gid", "release_groups.release_group.gid", - "series.series.gid", "works.work.gid"]), - F("name", ["areas.area.name", "artists.artist.name", "events.event.name", - "instruments.instrument.name", "labels.label.name", - "places.place.name", "recordings.recording.name", - "releases.release.name", "release_groups.release_group.name", - "series.series.name", "works.work.name"]), - F("text", "text"), - F("type", ["areas.__tablename__", "artists.__tablename__", - "events.__tablename__", "instruments.__tablename__", - "labels.__tablename__", "places.__tablename__", - "recordings.__tablename__", "releases.__tablename__", - "release_groups.__tablename__", "series.__tablename__", - "works.__tablename__"], - transformfunc=tfs.annotation_type) -], - 1.5, - convert.convert_annotation, - extraquery=queryext.filter_valid_annotations +from sir.schema import modelext, queryext +from sir.schema import transformfuncs as tfs +from sir.schema.searchentities import SearchEntity as E +from sir.schema.searchentities import SearchField as F + +SearchAnnotation = E( + modelext.CustomAnnotation, + [ + F("id", "id"), + F( + "entity", + [ + "areas.area.gid", + "artists.artist.gid", + "events.event.gid", + "instruments.instrument.gid", + "labels.label.gid", + "places.place.gid", + "recordings.recording.gid", + "releases.release.gid", + "release_groups.release_group.gid", + "series.series.gid", + "works.work.gid", + ], + ), + F( + "name", + [ + "areas.area.name", + "artists.artist.name", + "events.event.name", + "instruments.instrument.name", + "labels.label.name", + "places.place.name", + "recordings.recording.name", + "releases.release.name", + "release_groups.release_group.name", + "series.series.name", + "works.work.name", + ], + ), + F("text", "text"), + F( + "type", + [ + "areas.__tablename__", + "artists.__tablename__", + "events.__tablename__", + "instruments.__tablename__", + "labels.__tablename__", + "places.__tablename__", + "recordings.__tablename__", + "releases.__tablename__", + "release_groups.__tablename__", + "series.__tablename__", + "works.__tablename__", + ], + transformfunc=tfs.annotation_type, + ), + ], + 1.7, + extraquery=queryext.filter_valid_annotations, ) -SearchArea = E(modelext.CustomArea, [ - F("mbid", "gid"), - F("area", "name"), - F("alias", "aliases.name"), - F("comment", "comment"), - F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), - F("end", "end_date", transformfunc=tfs.index_partialdate_to_string), - F("ended", "ended", transformfunc=tfs.ended_to_string), - F("iso1", "iso_3166_1_codes.code"), - F("iso2", "iso_3166_2_codes.code"), - F("iso3", "iso_3166_3_codes.code"), - F("sortname", "aliases.sort_name"), - F("ref_count", ["place_count", "label_count", "artist_count"], transformfunc=tfs.integer_sum, trigger=False), - F("tag", "tags.tag.name"), - F("type", "type.name") -], - 1.5, - convert.convert_area, - extrapaths=["aliases.type.name", "aliases.type.id", - "aliases.sort_name", "aliases.type.gid", - "aliases.locale", "aliases.primary_for_locale", - "aliases.begin_date", "aliases.end_date", +SearchArea = E( + modelext.CustomArea, + [ + F("mbid", "gid"), + F("area", "name"), + F("comment", "comment"), + F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), + F("end", "end_date", transformfunc=tfs.index_partialdate_to_string), + F("ended", "ended", transformfunc=tfs.ended_to_string), + F("iso1", "iso_3166_1_codes.code"), + F("iso2", "iso_3166_2_codes.code"), + F("iso3", "iso_3166_3_codes.code"), + F("sortname", "aliases.sort_name"), + F( + "ref_count", + ["place_count", "label_count", "artist_count"], + transformfunc=tfs.integer_sum, + trigger=False, + ), + F("type", "type.name"), + F("type_gid", "type.gid"), + F("alias", "aliases.name", preserve_og=True), + F( + "alias_json", + [ + "aliases.name", + "aliases.sort_name", + "aliases.locale", + "aliases.primary_for_locale", + "aliases.begin_date", + "aliases.end_date", + "aliases.type.name", + "aliases.type.gid", + ], + objconverter=tfs.aliases_to_json, + ), + F("tag", "tags.tag.name", preserve_og=True), + F("tag_count", "tags.count", preserve_og=True), + F( + "rel_json", + [ "area_links.entity0.name", "area_links.entity0.gid", "area_links.entity0.begin_date", "area_links.entity0.end_date", "area_links.entity0.ended", - "area_links.entity0.type.id", "area_links.entity0.type.gid", "area_links.entity0.type.name", "area_links.link.link_type.name", "area_links.link.link_type.gid", - "area_links.link.attributes.attribute_type.name", - "area_links.link.attributes.attribute_type.gid", - "tags.count", "type.gid" - ] + ], + objconverter=tfs.area_relations_to_json, + ), + ], + 1.7, ) @@ -107,481 +158,692 @@ F("mbid", "gid"), F("artist", "name"), F("sortname", "sort_name"), - F("alias", "aliases.name"), - # Does not require a trigger since this will get updated on an alias update - F("primary_alias", "primary_aliases", trigger=False), + F("comment", "comment"), + + F("type", "type.name"), + F("type_gid", "type.gid"), + + F("gender_name", "gender.name", transformfunc=tfs.str_list_to_lowercase), + F("gender_gid", "gender.gid"), + F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), F("end", "end_date", transformfunc=tfs.index_partialdate_to_string), F("ended", "ended", transformfunc=tfs.ended_to_string), + + F("country", "area.iso_3166_1_codes.code"), + F("area", ["area.name", "area.aliases.name"]), + F("area_name", "area.name"), + F("area_gid", "area.gid"), + F("area_begindate", "area.begin_date", transformfunc=tfs.index_partialdate_to_string), + F("area_enddate", "area.end_date", transformfunc=tfs.index_partialdate_to_string), + F("area_ended", "area.ended", transformfunc=tfs.ended_to_string), + F("area_type", "area.type.name"), + F("area_type_gid", "area.type.gid"), + F("beginarea", ["begin_area.name", "begin_area.aliases.name"]), - F("country", "area.iso_3166_1_codes.code"), + F("beginarea_name", "begin_area.name"), + F("beginarea_gid", "begin_area.gid"), + F("beginarea_begindate", "begin_area.begin_date", transformfunc=tfs.index_partialdate_to_string), + F("beginarea_enddate", "begin_area.end_date", transformfunc=tfs.index_partialdate_to_string), + F("beginarea_ended", "begin_area.ended", transformfunc=tfs.ended_to_string), + F("beginarea_type", "begin_area.type.name"), + F("beginarea_type_gid", "begin_area.type.gid"), + F("endarea", ["end_area.name", "end_area.aliases.name"]), - F("ref_count", "artist_credit_names.artist_credit.ref_count", - transformfunc=sum, trigger=False), - F("comment", "comment"), - F("gender", "gender.name"), + F("endarea_name", "end_area.name"), + F("endarea_gid", "end_area.gid"), + F("endarea_begindate", "end_area.begin_date", transformfunc=tfs.index_partialdate_to_string), + F("endarea_enddate", "end_area.end_date", transformfunc=tfs.index_partialdate_to_string), + F("endarea_ended", "end_area.ended", transformfunc=tfs.ended_to_string), + F("endarea_type", "end_area.type.name"), + F("endarea_type_gid", "end_area.type.gid"), + + F("alias", "aliases.name", preserve_og=True), + F("alias_json", + ["aliases.name", "aliases.sort_name", "aliases.locale", + "aliases.primary_for_locale", "aliases.begin_date", "aliases.end_date", + "aliases.type.name", "aliases.type.gid"], + objconverter=tfs.aliases_to_json), + # Does not require a trigger since this will get updated on an alias update + F("primary_alias", "primary_aliases", trigger=False), + F("ipi", "ipis.ipi"), F("isni", "isnis.isni"), - F("tag", "tags.tag.name"), - F("type", "type.name") + + F("tag", "tags.tag.name", preserve_og=True), + F("tag_count", "tags.count", preserve_og=True), + + F("ref_count", "artist_credit_names.artist_credit.ref_count", + transformfunc=sum, trigger=False), ], - 1.5, - convert.convert_artist, - extrapaths=["tags.count", - "aliases.type.name", "aliases.type.id", - "aliases.type.gid", "aliases.sort_name", - "aliases.locale", "aliases.primary_for_locale", - "aliases.begin_date", "aliases.end_date", - "begin_area.gid", "area.gid", "end_area.gid", - "area.begin_date", "area.end_date", "area.ended", - "begin_area.begin_date", "begin_area.end_date", - "begin_area.ended", "end_area.begin_date", - "end_area.end_date", "end_area.ended", - "gender.gid", "area.type.gid", "area.type.name", - "begin_area.type.gid", "begin_area.type.name", - "end_area.type.gid", "end_area.type.name", - "type.gid"] + 1.7, ) -SearchCDStub = E(modelext.CustomReleaseRaw, [ - F("id", "id"), - F("title", "title"), - F("artist", "artist"), - F("comment", "comment"), - F("barcode", "barcode"), - F("added", "added", transformfunc=tfs.datetime_to_timestamp), - F("tracks", "discids.track_count"), - F("discid", "discids.discid") -], - 1.5, - convert.convert_cdstub +SearchCDStub = E( + modelext.CustomReleaseRaw, + [ + F("id", "id"), + F("title", "title"), + F("artist", "artist"), + F("comment", "comment"), + F("barcode", "barcode"), + F("added", "added", transformfunc=tfs.datetime_to_timestamp), + F("tracks", "discids.track_count"), + F("discid", "discids.discid"), + ], + 1.7, ) -SearchEditor = E(models.Editor, [ - F("id", "id"), - F("bio", "bio"), - F("editor", "name") -], - 1.5, - convert.convert_editor + +SearchEditor = E( + models.Editor, + [F("id", "id"), F("bio", "bio"), F("editor", "name")], + 1.7, ) -SearchEvent = E(modelext.CustomEvent, [ - F("mbid", "gid"), - F("alias", "aliases.name"), - F("aid", "area_links.entity0.gid"), - F("area", "area_links.entity0.name"), - F("arid", "artist_links.entity0.gid", transformfunc=tfs.uuid_set_to_str_set), - F("artist", "artist_links.entity0.name"), - F("pid", "place_links.entity1.gid"), - F("place", "place_links.entity1.name"), - F("comment", "comment"), - F("event", "name"), - F("tag", "tags.tag.name"), - F("type", "type.name"), - F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), - F("ended", "ended", transformfunc=tfs.ended_to_string), - F("end", "end_date", transformfunc=tfs.index_partialdate_to_string) -], - 1.5, - convert.convert_event, - extrapaths=["aliases.type.name", - "aliases.type.id", - "aliases.type.gid", + +SearchEvent = E( + modelext.CustomEvent, + [ + F("mbid", "gid"), + F("event", "name"), + F("comment", "comment"), + F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), + F("end", "end_date", transformfunc=tfs.index_partialdate_to_string), + F("ended", "ended", transformfunc=tfs.ended_to_string), + F("time", "time", transformfunc=tfs.index_time_to_string), + F("type", "type.name"), + F("type_gid", "type.gid"), + F("alias", "aliases.name", preserve_og=True), + F( + "alias_json", + [ + "aliases.name", "aliases.sort_name", "aliases.locale", "aliases.primary_for_locale", "aliases.begin_date", "aliases.end_date", - "area_links.entity0.name", + "aliases.type.name", + "aliases.type.gid", + ], + objconverter=tfs.aliases_to_json, + ), + F("tag", "tags.tag.name", preserve_og=True), + F("tag_count", "tags.count", preserve_og=True), + F("aid", "area_links.entity0.gid"), + F("area", "area_links.entity0.name"), + F("arid", "artist_links.entity0.gid", transformfunc=tfs.uuid_list_to_str_list), + F("artist", "artist_links.entity0.name"), + F("pid", "place_links.entity1.gid"), + F("place", "place_links.entity1.name"), + F( + "rel_json", + [ "area_links.entity0.gid", + "area_links.entity0.name", "area_links.link.link_type.name", "area_links.link.link_type.gid", - "area_links.link.attributes.attribute_type.name", - "area_links.link.attributes.attribute_type.gid", "artist_links.entity0.gid", "artist_links.entity0.name", - "artist_links.entity0.comment", "artist_links.entity0.sort_name", + "artist_links.entity0.comment", "artist_links.link.link_type.name", "artist_links.link.link_type.gid", - "artist_links.link.attributes.attribute_type.name", - "artist_links.link.attributes.attribute_type.gid", "place_links.entity1.gid", "place_links.entity1.name", "place_links.link.link_type.name", "place_links.link.link_type.gid", - "place_links.link.attributes.attribute_type.name", - "place_links.link.attributes.attribute_type.gid", - "tags.count", "type.gid", - "time"] + ], + objconverter=tfs.event_relations_to_json, + ), + ], + 1.7, ) -SearchInstrument = E(modelext.CustomInstrument, [ - F("alias", "aliases.name"), - F("comment", "comment"), - F("description", "description"), - F("mbid", "gid"), - F("instrument", "name"), - F("tag", "tags.tag.name"), - F("type", "type.name") -], - 1.5, - convert.convert_instrument, - extrapaths=["aliases.type.name", "aliases.type.id", - "aliases.sort_name", "aliases.type.gid", - "aliases.locale", "aliases.primary_for_locale", - "aliases.begin_date", "aliases.end_date", "tags.count", "type.gid"] -) +SearchInstrument = E( + modelext.CustomInstrument, + [ + F("alias", "aliases.name"), + F("comment", "comment"), + F("description", "description"), + F("mbid", "gid"), + F("instrument", "name"), + F("tag", "tags.tag.name", preserve_og=True), + F("tag_count", "tags.count", preserve_og=True), + F("type", "type.name"), + F("type_gid", "type.gid"), + F( + "alias_json", + [ + "aliases.type.name", + "aliases.type.id", + "aliases.sort_name", + "aliases.type.gid", + "aliases.locale", + "aliases.primary_for_locale", + "aliases.begin_date", + "aliases.end_date", + ], + objconverter=tfs.aliases_to_json, + ), + ], + 1.7, +) -SearchLabel = E(modelext.CustomLabel, [ - F("mbid", "gid"), - F("label", "name"), - F("alias", "aliases.name"), - F("area", ["area.name", "area.aliases.name"]), - F("country", "area.iso_3166_1_codes.code"), - F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), - F("end", "end_date", transformfunc=tfs.index_partialdate_to_string), - F("ended", "ended", transformfunc=tfs.ended_to_string), - F("code", "label_code"), - F("comment", "comment"), - F("release_count", "release_count", transformfunc=tfs.integer_sum, trigger=False), - F("sortname", "aliases.sort_name"), - F("ipi", "ipis.ipi"), - F("isni", "isnis.isni"), - F("tag", "tags.tag.name"), - F("type", "type.name") -], - 1.5, - convert.convert_label, - extrapaths=["aliases.type.name", "aliases.type.id", - "aliases.type.gid", "aliases.sort_name", - "aliases.locale", "aliases.primary_for_locale", - "aliases.begin_date", "aliases.end_date", - "area.gid", "area.type.name", "area.type.gid", - "area.begin_date", "area.end_date", "area.ended", - "tags.count", "type.gid" - ] +SearchLabel = E( + modelext.CustomLabel, + [ + F("mbid", "gid"), + F("label", "name"), + F("sortname", "aliases.sort_name"), + F("type", "type.name"), + F("type_gid", "type.gid"), + F("area", ["area.name", "area.aliases.name"]), + F("area_name", "area.name"), + F("area_gid", "area.gid"), + F("area_begindate", "area.begin_date", transformfunc=tfs.index_partialdate_to_string), + F("area_enddate", "area.end_date", transformfunc=tfs.index_partialdate_to_string), + F("area_ended", "area.ended", transformfunc=tfs.ended_to_string), + F("area_type_gid", "area.type.gid"), + F("area_type", "area.type.name"), + F("country", "area.iso_3166_1_codes.code"), + F("code", "label_code"), + F("alias", "aliases.name"), + F( + "alias_json", + [ + "aliases.type.name", + "aliases.type.id", + "aliases.type.gid", + "aliases.sort_name", + "aliases.locale", + "aliases.primary_for_locale", + "aliases.begin_date", + "aliases.end_date", + ], + objconverter=tfs.aliases_to_json, + ), + F("ipi", "ipis.ipi"), + F("isni", "isnis.isni"), + F("comment", "comment"), + F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), + F("end", "end_date", transformfunc=tfs.index_partialdate_to_string), + F("ended", "ended", transformfunc=tfs.ended_to_string), + F("tag", "tags.tag.name", preserve_og=True), + F("tag_count", "tags.count", preserve_og=True), + F("release_count", "release_count", transformfunc=tfs.integer_sum, trigger=False), + ], + 1.7, ) -SearchPlace = E(modelext.CustomPlace, [ - F("mbid", "gid"), - F("address", "address"), - F("alias", "aliases.name"), - F("area", ["area.name", "area.aliases.name"]), - F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), - F("comment", "comment"), - F("end", "end_date", transformfunc=tfs.index_partialdate_to_string), - F("ended", "ended", transformfunc=tfs.ended_to_string), - F("lat", "coordinates", transformfunc=tfs.lat), - F("long", "coordinates", transformfunc=tfs.long), - F("place", "name"), - F("type", "type.name") -], - 1.5, - convert.convert_place, - extrapaths=["aliases.type.name", "aliases.type.id", - "aliases.type.gid", "aliases.sort_name", - "aliases.locale", "aliases.primary_for_locale", - "aliases.begin_date", "aliases.end_date", - "area.gid", "area.type.gid", "area.type.name", - "area.begin_date", "area.end_date", "area.ended", - "type.gid"] +SearchPlace = E( + modelext.CustomPlace, + [ + F("mbid", "gid"), + F("place", "name"), + F("address", "address"), + F("comment", "comment"), + F("type", "type.name"), + F("type_gid", "type.gid"), + F("area", ["area.name", "area.aliases.name"]), + F("area_name", "area.name"), + F("area_gid", "area.gid"), + F("area_begindate", "area.begin_date", transformfunc=tfs.index_partialdate_to_string), + F("area_enddate", "area.end_date", transformfunc=tfs.index_partialdate_to_string), + F("area_ended", "area.ended", transformfunc=tfs.ended_to_string), + F("area_type_gid", "area.type.gid"), + F("area_type", "area.type.name"), + F("alias", "aliases.name"), + F( + "alias_json", + [ + "aliases.type.name", + "aliases.type.id", + "aliases.type.gid", + "aliases.sort_name", + "aliases.locale", + "aliases.primary_for_locale", + "aliases.begin_date", + "aliases.end_date", + ], + objconverter=tfs.aliases_to_json, + ), + F("begin", "begin_date", transformfunc=tfs.index_partialdate_to_string), + F("end", "end_date", transformfunc=tfs.index_partialdate_to_string), + F("ended", "ended", transformfunc=tfs.ended_to_string), + F("lat", "coordinates", transformfunc=tfs.lat), + F("long", "coordinates", transformfunc=tfs.long), + ], + 1.7, ) -SearchRecording = E(modelext.CustomRecording, [ - F("alias", "aliases.name"), - F("arid", "artist_credit.artists.artist.gid", transformfunc=tfs.uuid_set_to_str_set), - F("artist", "artist_credit.name"), - F("artistname", "artist_credit.artists.artist.name"), - F("comment", "comment"), - F("country", "tracks.medium.release.country_dates.country.area.iso_3166_1_codes.code"), - F("creditname", "artist_credit.artists.name"), - F("date", "tracks.medium.release.country_dates.date", - transformfunc=tfs.index_partialdate_to_string), - F("dur", "length"), - F("firstreleasedate", "first_release_date.date", - transformfunc=tfs.index_partialdate_to_string), - F("format", "tracks.medium.format.name"), - F("isrc", "isrcs.isrc"), - F("mbid", "gid"), - F("number", "tracks.number"), - F("position", "tracks.medium.position"), - F("primarytype", "tracks.medium.release.release_group.type.name"), - F("qdur", "length", transformfunc=tfs.qdur), - F("recording", "name"), - F("reid", "tracks.medium.release.gid"), - F("release", "tracks.medium.release.name"), - F("rgid", "tracks.medium.release.release_group.gid"), - F("secondarytype", - "tracks.medium.release.release_group.secondary_types.secondary_type." - "name"), - F("status", "tracks.medium.release.status.name"), - F("tag", "tags.tag.name"), - F("tid", "tracks.gid"), - F("tnum", "tracks.position"), - F("tracks", "tracks.medium.track_count"), - F("tracksrelease", "tracks.medium.release.mediums.track_count", - transformfunc=sum), - F("video", "video", transformfunc=tfs.boolean) -], - 1.5, - convert.convert_recording, - extrapaths=["artist_credit.artists.artist.aliases.begin_date", - "artist_credit.artists.artist.aliases.end_date", - "artist_credit.artists.artist.aliases.locale", +SearchRecording = E( + modelext.CustomRecording, + [ + F("mbid", "gid"), + F("recording", "name"), + F("comment", "comment"), + F("alias", "aliases.name"), + F("arid", "artist_credit.artists.artist.gid", transformfunc=tfs.uuid_list_to_str_list), + F("artist", "artist_credit.name"), + F("artistname", "artist_credit.artists.artist.name"), + F("creditname", "artist_credit.artists.name"), + F( + "artist_credit_json", + [ + "artist_credit.gid", + "artist_credit.artists.name", + "artist_credit.artists.join_phrase", + "artist_credit.artists.artist.gid", + "artist_credit.artists.artist.name", + "artist_credit.artists.artist.comment", + "artist_credit.artists.artist.sort_name", "artist_credit.artists.artist.aliases.name", - "artist_credit.artists.artist.aliases.primary_for_locale", "artist_credit.artists.artist.aliases.sort_name", - "artist_credit.artists.artist.aliases.type.id", + "artist_credit.artists.artist.aliases.locale", + "artist_credit.artists.artist.aliases.primary_for_locale", + "artist_credit.artists.artist.aliases.begin_date", + "artist_credit.artists.artist.aliases.end_date", "artist_credit.artists.artist.aliases.type.name", + "artist_credit.artists.artist.aliases.type.id", "artist_credit.artists.artist.aliases.type.gid", - "artist_credit.artists.artist.comment", - "artist_credit.artists.artist.gid", - "artist_credit.artists.artist.name", - "artist_credit.artists.artist.sort_name", - "artist_credit.artists.join_phrase", - "artist_credit.artists.name", - "artist_credit.gid", - "artist_credit.name", - "tags.count", - "tags.tag.name", + ], + objconverter=tfs.recording_artist_credit_to_json, + ), + F("dur", "length"), + F("qdur", "length", transformfunc=tfs.qdur), + F( + "firstreleasedate", + "first_release_date.date", + transformfunc=tfs.index_partialdate_to_string, + ), + F("isrc", "isrcs.isrc"), + F("video", "video", transformfunc=tfs.boolean), + F("country", "tracks.medium.release.country_dates.country.area.iso_3166_1_codes.code"), + F( + "date", + "tracks.medium.release.country_dates.date", + transformfunc=tfs.index_partialdate_to_string, + ), + F("format", "tracks.medium.format.name"), + F("number", "tracks.number"), + F("position", "tracks.medium.position"), + F("primarytype", "tracks.medium.release.release_group.type.name"), + F("reid", "tracks.medium.release.gid"), + F("release", "tracks.medium.release.name"), + F("rgid", "tracks.medium.release.release_group.gid"), + F( + "secondarytype", + "tracks.medium.release.release_group.secondary_types.secondary_type.name", + ), + F("status", "tracks.medium.release.status.name"), + F("tid", "tracks.gid"), + F("tnum", "tracks.position"), + F("tracks", "tracks.medium.track_count"), + F("tracksrelease", "tracks.medium.release.mediums.track_count", transformfunc=sum), + F( + "release_json", + [ + "tracks.gid", + "tracks.name", + "tracks.number", + "tracks.position", "tracks.length", "tracks.medium.gid", - "tracks.medium.cdtocs.id", - "tracks.medium.release.artist_credit.artists.artist.comment", + "tracks.medium.position", + "tracks.medium.track_count", + "tracks.medium.format.name", + "tracks.medium.release.gid", + "tracks.medium.release.name", + "tracks.medium.release.comment", + "tracks.medium.release.artist_credit.gid", + "tracks.medium.release.artist_credit.artists.name", + "tracks.medium.release.artist_credit.artists.join_phrase", "tracks.medium.release.artist_credit.artists.artist.gid", "tracks.medium.release.artist_credit.artists.artist.name", + "tracks.medium.release.artist_credit.artists.artist.comment", "tracks.medium.release.artist_credit.artists.artist.sort_name", - "tracks.medium.release.artist_credit.artists.join_phrase", - "tracks.medium.release.artist_credit.artists.name", - "tracks.medium.release.artist_credit.gid", - "tracks.medium.release.artist_credit.name", - "tracks.medium.release.comment", + "tracks.medium.release.country_dates.date", "tracks.medium.release.country_dates.country.area.gid", - "tracks.medium.release.country_dates.country.area." - "iso_3166_1_codes.code", "tracks.medium.release.country_dates.country.area.name", - "tracks.medium.release.country_dates.date_day", - "tracks.medium.release.country_dates.date_month", - "tracks.medium.release.country_dates.date_year", - "tracks.medium.release.release_group.comment", + "tracks.medium.release.country_dates.country.area.iso_3166_1_codes.code", + "tracks.medium.release.mediums.track_count", + "tracks.medium.release.release_group.gid", "tracks.medium.release.release_group.name", + "tracks.medium.release.release_group.comment", + "tracks.medium.release.release_group.type.name", "tracks.medium.release.release_group.type.gid", + "tracks.medium.release.release_group.secondary_types.secondary_type.name", "tracks.medium.release.release_group.secondary_types.secondary_type.gid", + "tracks.medium.release.status.name", "tracks.medium.release.status.gid", - "tracks.name"] + ], + objconverter=tfs.recording_releases_to_json, + ), + F("tag", "tags.tag.name", preserve_og=True), + F("tag_count", "tags.count", preserve_og=True), + ], + 1.7, ) -SearchRelease = E(modelext.CustomRelease, [ - F("mbid", "gid"), - F("release", "name"), - F("alias", "aliases.name"), - F("arid", "artist_credit.artists.artist.gid", transformfunc=tfs.uuid_set_to_str_set), - F("artist", "artist_credit.name"), - F("artistname", "artist_credit.artists.artist.name"), - F("asin", "asin.amazon_asin"), - F("creditname", "artist_credit.artists.name"), - F("country", "country_dates.country.area.iso_3166_1_codes.code"), - F("date", "country_dates.date", - transformfunc=tfs.index_partialdate_to_string), - F("barcode", "barcode", transformfunc=tfs.fill_none), - F("catno", "labels.catalog_number"), - F("comment", "comment"), - F("format", "mediums.format.name"), - F("laid", "labels.label.gid"), - F("label", "labels.label.name"), - F("lang", "language.iso_code_3"), - F("mediums", "medium_count", transformfunc=tfs.integer_sum, trigger=False), - F("mediumid", "mediums.gid"), - F("packaging", "packaging.name"), - F("primarytype", "release_group.type.name"), - F("quality", "quality"), - F("rgid", "release_group.gid"), - F("script", "script.iso_code"), - F("secondarytype", "release_group.secondary_types.secondary_type.name"), - F("status", "status.name"), - F("tracks", "mediums.track_count", - transformfunc=sum), - F("tracksmedium", "mediums.track_count"), - F("tag", "tags.tag.name") -], - 1.5, - convert.convert_release, - extrapaths=["artist_credit.artists.join_phrase", - "artist_credit.artists.artist.aliases.begin_date", - "artist_credit.artists.artist.aliases.end_date", - "artist_credit.artists.artist.aliases.locale", - "artist_credit.artists.artist.aliases.name", - "artist_credit.artists.artist.aliases.primary_for_locale", - "artist_credit.artists.artist.aliases.sort_name", - "artist_credit.artists.artist.aliases.type.id", - "artist_credit.artists.artist.aliases.type.name", - "artist_credit.artists.artist.aliases.type.gid", +SearchRelease = E( + modelext.CustomRelease, + [ + F("mbid", "gid"), + F("release", "name"), + F("alias", "aliases.name"), + F("arid", "artist_credit.artists.artist.gid", transformfunc=tfs.uuid_list_to_str_list), + F("artist", "artist_credit.name"), + F("artistname", "artist_credit.artists.artist.name"), + F("creditname", "artist_credit.artists.name"), + F( + "artist_credit_json", + [ + "artist_credit.gid", + "artist_credit.artists.name", + "artist_credit.artists.join_phrase", "artist_credit.artists.artist.gid", - "artist_credit.artists.artist.sort_name", + "artist_credit.artists.artist.name", "artist_credit.artists.artist.comment", - "artist_credit.gid", + "artist_credit.artists.artist.sort_name", + ], + objconverter=tfs.release_artist_credit_to_json, + ), + F("asin", "asin.amazon_asin"), + F("barcode", "barcode", transformfunc=tfs.fill_none), + F("barcode_none", "barcode", objconverter=tfs.release_barcode_none), + F("comment", "comment"), + F("packaging", "packaging.name"), + F("packaging_gid", "packaging.gid"), + F("country", "country_dates.country.area.iso_3166_1_codes.code"), + F("date", "country_dates.date", transformfunc=tfs.index_partialdate_to_string), + F( + "release_event_json", + [ + "country_dates.date", "country_dates.country.area.gid", "country_dates.country.area.name", "country_dates.country.area.iso_3166_1_codes.code", - "country_dates.date_day", - "country_dates.date_month", - "country_dates.date_year", - "mediums.cdtocs.id", - "release_group.comment", - "release_group.name", + ], + objconverter=tfs.release_events_to_json, + ), + F("catno", "labels.catalog_number"), + F("laid", "labels.label.gid"), + F("label", "labels.label.name"), + F( + "label_info_json", + ["labels.catalog_number", "labels.label.gid", "labels.label.name"], + objconverter=tfs.release_label_info_to_json, + ), + F("format", "mediums.format.name"), + F("mediumid", "mediums.gid"), + F("mediums", "medium_count", transformfunc=tfs.integer_sum, trigger=False), + F("tracks", "mediums.track_count", transformfunc=sum), + F("tracksmedium", "mediums.track_count", preserve_og=True), + F( + "medium_json", + ["mediums.gid", "mediums.format.name", "mediums.cdtocs.id", "mediums.track_count"], + objconverter=tfs.release_mediums_to_json, + ), + F("rgid", "release_group.gid"), + F("rg_name", "release_group.name"), + F("rg_comment", "release_group.comment"), + F("primarytype", "release_group.type.name"), + F("primarytype_gid", "release_group.type.gid"), + F("secondarytype", "release_group.secondary_types.secondary_type.name", preserve_og=True), + F( + "secondarytype_gid", + "release_group.secondary_types.secondary_type.gid", + preserve_og=True, + ), + F( + "calc_type", + [ + "release_group.type.name", + "release_group.type.gid", + "release_group.secondary_types.secondary_type.name", + "release_group.secondary_types.secondary_type.gid", + ], + objconverter=tfs.release_calc_type, + ), + F( + "calc_type_gid", + [ + "release_group.type.name", "release_group.type.gid", + "release_group.secondary_types.secondary_type.name", "release_group.secondary_types.secondary_type.gid", - "status.gid", - "packaging.gid", - "language.iso_code_3", - "tags.count"] + ], + objconverter=tfs.release_calc_type_gid, + ), + F("status", "status.name"), + F("status_gid", "status.gid"), + F("lang", "language.iso_code_3"), + F("script", "script.iso_code"), + F("quality", "quality"), + F("tag", "tags.tag.name", preserve_og=True), + F("tag_count", "tags.count", preserve_og=True), + ], + 1.7, ) -SearchReleaseGroup = E(modelext.CustomReleaseGroup, [ - F("mbid", "gid"), - F("releasegroup", "name"), - F("alias", "aliases.name"), - F("arid", "artist_credit.artists.artist.gid", transformfunc=tfs.uuid_set_to_str_set), - F("artist", "artist_credit.name"), - F("artistname", "artist_credit.artists.artist.name"), - F("creditname", "artist_credit.artists.name"), - F("firstreleasedate", "first_release_date.first_release_date", - transformfunc=tfs.index_partialdate_to_string), - F("release", "releases.name"), - F("reid", "releases.gid"), - F("releases", "release_count", transformfunc=tfs.integer_sum, trigger=False), - F("status", "releases.status.name"), - F("comment", "comment"), - F("tag", "tags.tag.name"), - F("primarytype", "type.name"), - F("secondarytype", "secondary_types.secondary_type.name") -], - 1.5, - convert.convert_release_group, - extrapaths=["artist_credit.artists.join_phrase", - "artist_credit.artists.artist.aliases.begin_date", - "artist_credit.artists.artist.aliases.end_date", - "artist_credit.artists.artist.aliases.locale", +SearchReleaseGroup = E( + modelext.CustomReleaseGroup, + [ + F("mbid", "gid"), + F("releasegroup", "name"), + F("comment", "comment"), + F("arid", "artist_credit.artists.artist.gid", transformfunc=tfs.uuid_list_to_str_list), + F("artist", "artist_credit.name"), + F("artistname", "artist_credit.artists.artist.name"), + F("creditname", "artist_credit.artists.name"), + F( + "artist_credit_json", + [ + "artist_credit.gid", + "artist_credit.name", + "artist_credit.artists.name", + "artist_credit.artists.join_phrase", + "artist_credit.artists.artist.gid", + "artist_credit.artists.artist.name", + "artist_credit.artists.artist.comment", + "artist_credit.artists.artist.sort_name", "artist_credit.artists.artist.aliases.name", - "artist_credit.artists.artist.aliases.primary_for_locale", "artist_credit.artists.artist.aliases.sort_name", + "artist_credit.artists.artist.aliases.locale", + "artist_credit.artists.artist.aliases.primary_for_locale", + "artist_credit.artists.artist.aliases.begin_date", + "artist_credit.artists.artist.aliases.end_date", + "artist_credit.artists.artist.aliases.type.name", "artist_credit.artists.artist.aliases.type.id", "artist_credit.artists.artist.aliases.type.gid", - "artist_credit.artists.artist.aliases.type.name", - "artist_credit.artists.artist.gid", - "artist_credit.artists.artist.sort_name", - "artist_credit.artists.artist.comment", - "artist_credit.gid", - "tags.count", "type.gid", - "releases.status.gid", - "secondary_types.secondary_type.gid" - ] + ], + objconverter=tfs.release_group_artist_credit_to_json, + ), + F("alias", "aliases.name"), + F( + "firstreleasedate", + "first_release_date.first_release_date", + transformfunc=tfs.index_partialdate_to_string, + ), + F("primarytype", "type.name"), + F("primarytype_gid", "type.gid"), + F("secondarytype", "secondary_types.secondary_type.name", preserve_og=True), + F("secondarytype_gid", "secondary_types.secondary_type.gid", preserve_og=True), + F( + "calc_type", + [ + "type.name", + "type.gid", + "secondary_types.secondary_type.name", + "secondary_types.secondary_type.gid", + ], + objconverter=tfs.release_group_calc_type, + ), + F( + "calc_type_gid", + [ + "type.name", + "type.gid", + "secondary_types.secondary_type.name", + "secondary_types.secondary_type.gid", + ], + objconverter=tfs.release_group_calc_type_gid, + ), + F("release", "releases.name"), + F("reid", "releases.gid"), + F("status", "releases.status.name"), + F( + "release_json", + ["releases.gid", "releases.name", "releases.status.name", "releases.status.gid"], + objconverter=tfs.release_group_releases_to_json, + ), + F("releases", "release_count", transformfunc=tfs.integer_sum, trigger=False), + F("tag", "tags.tag.name", preserve_og=True), + F("tag_count", "tags.count", preserve_og=True), + ], + 1.7, ) -SearchSeries = E(modelext.CustomSeries, [ - F("mbid", "gid"), - F("alias", "aliases.name"), - F("comment", "comment"), - F("series", "name"), - F("tag", "tags.tag.name"), - F("type", "type.name") -], - 1.5, - convert.convert_series, - extrapaths=["tags.count", - "aliases.type.name", "aliases.type.id", - "aliases.type.gid", "aliases.sort_name", - "aliases.locale", "aliases.primary_for_locale", - "aliases.begin_date", "aliases.end_date", "type.gid"] +SearchSeries = E( + modelext.CustomSeries, + [ + F("mbid", "gid"), + F("series", "name"), + F("type", "type.name"), + F("type_gid", "type.gid"), + F("alias", "aliases.name"), + F( + "alias_json", + [ + "aliases.type.name", + "aliases.type.id", + "aliases.type.gid", + "aliases.sort_name", + "aliases.locale", + "aliases.primary_for_locale", + "aliases.begin_date", + "aliases.end_date", + ], + objconverter=tfs.aliases_to_json, + ), + F("comment", "comment"), + F("tag", "tags.tag.name", preserve_og=True), + F("tag_count", "tags.count", preserve_og=True), + ], + 1.7, ) -SearchTag = E(models.Tag, [ - F("id", "id"), - F("tag", "name") -], - 1.5, - convert.convert_standalone_tag +SearchTag = E( + models.Tag, + [F("id", "id"), F("tag", "name")], + 1.7, ) -SearchUrl = E(modelext.CustomURL, [ - F("mbid", "gid"), - F("url", "url"), - F("relationtype", ["artist_links.link.link_type.name", - "release_links.link.link_type.name"]), - F("targetid", ["artist_links.entity0.gid", - "release_links.entity0.gid"]), - F("targettype", ["artist_links.__tablename__", - "release_links.__tablename__"], - transformfunc=tfs.url_type), -], - 1.5, - convert.convert_url, - extrapaths=["artist_links.entity0.gid", +SearchUrl = E( + modelext.CustomURL, + [ + F("mbid", "gid"), + F("url", "url"), + F( + "relationtype", + ["artist_links.link.link_type.name", "release_links.link.link_type.name"], + ), + F("targetid", ["artist_links.entity0.gid", "release_links.entity0.gid"]), + F( + "targettype", + ["artist_links.__tablename__", "release_links.__tablename__"], + transformfunc=tfs.url_type, + ), + F( + "rel_json", + [ + "artist_links.entity0.gid", "artist_links.entity0.name", "artist_links.entity0.comment", "artist_links.entity0.sort_name", "artist_links.link.link_type.name", "artist_links.link.link_type.gid", - "artist_links.link.attributes.attribute_type.name", - "artist_links.link.attributes.attribute_type.gid", "release_links.entity0.gid", "release_links.entity0.name", "release_links.entity0.comment", "release_links.link.link_type.name", "release_links.link.link_type.gid", - "release_links.link.attributes.attribute_type.name", - "release_links.link.attributes.attribute_type.gid", - ] + ], + objconverter=tfs.url_relations_to_json, + ), + ], + 1.7, ) -SearchWork = E(modelext.CustomWork, [ - F("mbid", "gid"), - F("work", "name"), - F("alias", "aliases.name"), - F("arid", "artist_links.entity0.gid", transformfunc=tfs.uuid_set_to_str_set), - F("artist", "artist_links.entity0.name"), - F("comment", "comment"), - F("iswc", "iswcs.iswc"), - F("lang", "languages.language.iso_code_3"), - F("recording", "recording_links.entity0.name"), - F("recording_count", "recording_links.entity0.gid", transformfunc=tfs.integer_count_all, trigger=False), - F("rid", "recording_links.entity0.gid"), - F("tag", "tags.tag.name"), - F("type", "type.name") -], - 1.5, - convert.convert_work, - extrapaths=["aliases.type.name", "aliases.type.id", +SearchWork = E( + modelext.CustomWork, + [ + F("mbid", "gid"), + F("work", "name"), + F("alias", "aliases.name"), + F( + "alias_json", + [ + "aliases.type.name", + "aliases.type.id", "aliases.type.gid", - "aliases.sort_name", "aliases.locale", + "aliases.sort_name", + "aliases.locale", "aliases.primary_for_locale", - "aliases.begin_date", "aliases.end_date", - "artist_links.entity0.sort_name", + "aliases.begin_date", + "aliases.end_date", + ], + objconverter=tfs.aliases_to_json, + ), + F("arid", "artist_links.entity0.gid", transformfunc=tfs.uuid_list_to_str_list), + F("artist", "artist_links.entity0.name"), + F("recording", "recording_links.entity0.name"), + F("rid", "recording_links.entity0.gid"), + F( + "rel_json", + [ + "artist_links.entity0.gid", + "artist_links.entity0.name", "artist_links.entity0.comment", + "artist_links.entity0.sort_name", "artist_links.link.link_type.name", "artist_links.link.link_type.gid", - "artist_links.link.attributes.attribute_type.name", - "artist_links.link.attributes.attribute_type.gid", + "recording_links.entity0.gid", + "recording_links.entity0.name", + "recording_links.entity0.video", "recording_links.link.link_type.name", "recording_links.link.link_type.gid", - "recording_links.link.attributes.attribute_type.name", - "recording_links.link.attributes.attribute_type.gid", - "recording_links.entity0.video", - "tags.count", "type.gid"] + ], + objconverter=tfs.work_relations_to_json, + ), + F("comment", "comment"), + F("iswc", "iswcs.iswc"), + F("lang", "languages.language.iso_code_3", preserve_og=True), + F("type", "type.name"), + F("type_gid", "type.gid"), + F("tag", "tags.tag.name"), + F( + "recording_count", + "recording_links.entity0.gid", + transformfunc=tfs.integer_count_all, + trigger=False, + ), + ], + 1.7, ) @@ -619,8 +881,11 @@ def generate_update_map(): :rtype (dict, dict, dict, dict) """ - from sir.query_generation.paths import (unique_split_paths, last_model_in_path, - second_last_model_in_path) + from sir.query_generation.paths import ( + last_model_in_path, + second_last_model_in_path, + unique_split_paths, + ) paths = defaultdict(set) column_map = defaultdict(set) diff --git a/sir/schema/searchentities.py b/sir/schema/searchentities.py index 01455b4b..2f4f11aa 100644 --- a/sir/schema/searchentities.py +++ b/sir/schema/searchentities.py @@ -117,7 +117,8 @@ class SearchField(object): 3. `gid` column from the `Area` class (model) """ - def __init__(self, name, paths, transformfunc=None, trigger=True): + def __init__(self, name, paths, transformfunc=None, trigger=True, preserve_og=False, + objconverter=None): """ :param str name: The name of the field. :param [str] paths: A dot-delimited path (or a list of them) along which @@ -129,6 +130,11 @@ def __init__(self, name, paths, transformfunc=None, trigger=True): :param method trigger: Whether changes to the path should trigger an update in the Solr core for the entity with this field. Defaults to `True`. + :param method preserve_og: Flag to preserve original values and order. + Defaults to `False`. + :param method objconverter: An optional function that builds this + field's value directly from the entity + object. """ self.name = name if not isinstance(paths, list): @@ -136,6 +142,8 @@ def __init__(self, name, paths, transformfunc=None, trigger=True): self.paths = paths self.transformfunc = transformfunc self.trigger = trigger + self.preserve_og = preserve_og + self.objconverter = objconverter class SearchEntity(object): @@ -261,33 +269,40 @@ def query_result_to_dict(self, obj): data = {} for field in self.fields: fieldname = field.name - tempvals = set() + + if field.objconverter is not None: + value = field.objconverter(obj) + if value: + data[fieldname] = value + continue + + tempvals = list() for path in field.paths: for value in iterate_path_values(path, obj): if value is not None: if isinstance(value, list): - tempvals.update(set(value)) + tempvals.extend(value) else: - tempvals.add(value) + tempvals.append(value) + else: + if field.preserve_og is True: + tempvals.append("") if field.transformfunc is not None: tempvals = field.transformfunc(tempvals) - if (isinstance(tempvals, set) or isinstance(tempvals, list)) and len(tempvals) == 1: - tempvals = tempvals.pop() - if tempvals is not None and tempvals: - if isinstance(tempvals, UUID): - new_tempvals = str(tempvals) - elif isinstance(tempvals, set) or isinstance(tempvals, list): - new_tempvals = list() - for tempval in tempvals: - if isinstance(tempval, UUID): - tempval = str(tempval) - new_tempvals.append(tempval) - else: - new_tempvals = tempvals - data[fieldname] = new_tempvals - if (config.CFG.getboolean("sir", "wscompat") and self.compatconverter is - not None): - data["_store"] = str(tostring(self.compatconverter(obj).to_etree(), encoding='us-ascii'), encoding='us-ascii') + if (isinstance(tempvals, (set, list))): + if field.preserve_og is False: + tempvals = list(set(tempvals)) + if len(tempvals) == 1: + tempvals = tempvals.pop() + + if tempvals is not None and (tempvals or isinstance(tempvals, int)): + if isinstance(tempvals, UUID): + tempvals = str(tempvals) + elif isinstance(tempvals, list): + for idx in range(len(tempvals)): + if isinstance(tempvals[idx], UUID): + tempvals[idx] = str(tempvals[idx]) + data[fieldname] = tempvals return data diff --git a/sir/schema/transformfuncs.py b/sir/schema/transformfuncs.py index e74bd89f..e9109ec5 100644 --- a/sir/schema/transformfuncs.py +++ b/sir/schema/transformfuncs.py @@ -1,9 +1,10 @@ # Copyright (c) 2014, 2015 Wieland Hoffmann # License: MIT, see LICENSE for details -from datetime import datetime +from functools import lru_cache -from sir.wscompat.convert import partialdate_to_string +import orjson +from sir.wscompat.convert import calculate_type, partialdate_to_string ANNOTATION_TABLE_TO_ENTITYTYPE = { "area_annotation": "area", @@ -16,7 +17,7 @@ "release_annotation": "release", "release_group_annotation": "release-group", "series_annotation": "series", - "work_annotation": "work" + "work_annotation": "work", } URL_LINK_TABLE_TO_ENTITYTYPE = { @@ -31,7 +32,7 @@ "l_release_group_url": "release-group", "l_series_url": "series", "l_url_work": "work", - "l_url_url": "url" + "l_url_url": "url", } @@ -41,7 +42,7 @@ def fill_none(values): # in the DB as an empty string, so doing this allows us to search for releases with # `[none]` type barcode via the syntax `barcode:none` if "" in values: - return values.add('none') + return values.append("none") return values @@ -71,6 +72,393 @@ def index_partialdate_to_string(dates): return None +def index_partialdatelist_to_string(date_list): + for idx in range(len(date_list)): + date_list[idx] = partialdate_to_string(date_list[idx]) + return date_list + +def index_time_to_string(times): + if len(times): + return times.pop().strftime("%H:%M:%S") + return None + + +def _alias_dict(alias) -> dict: + alias_dict = {"name": alias.name} + if alias.sort_name: + alias_dict["sort_name"] = alias.sort_name + if alias.locale: + alias_dict["locale"] = alias.locale + if alias.primary_for_locale: + alias_dict["primary"] = "primary" + begin_date = partialdate_to_string(alias.begin_date) + if begin_date: + alias_dict["begin_date"] = begin_date + end_date = partialdate_to_string(alias.end_date) + if end_date: + alias_dict["end_date"] = end_date + if alias.type is not None: + alias_dict["type"] = alias.type.name + alias_dict["type_id"] = str(alias.type.gid) + return alias_dict + + +def aliases_to_json(artist): + return [orjson.dumps(_alias_dict(a)).decode("utf-8") for a in artist.aliases] + + +def area_relations_to_json(area) -> str: + relations = [] + for link in area.area_links: + parent = link.entity0 + parent_dict = {"id": str(parent.gid), "name": parent.name} + if parent.type is not None: + parent_dict["type"] = parent.type.name + parent_dict["type_id"] = str(parent.type.gid) + begin_date = partialdate_to_string(parent.begin_date) + if begin_date: + parent_dict["begin_date"] = begin_date + end_date = partialdate_to_string(parent.end_date) + if end_date: + parent_dict["end_date"] = end_date + parent_dict["ended"] = "true" if parent.ended else "false" + + data = { + "direction": "backward", + "type": link.link.link_type.name, + "type_id": str(link.link.link_type.gid), + "target": str(parent.gid), + "area": parent_dict, + } + relations.append(orjson.dumps(data).decode("utf-8")) + return relations + + +def url_relations_to_json(url): + relations = [] + for link in url.artist_links: + artist = link.artist + artist_dict = {"id": str(artist.gid), "name": artist.name} + if artist.comment: + artist_dict["disambiguation"] = artist.comment + if artist.sort_name is not None: + artist_dict["sort_name"] = artist.sort_name + relations.append( + orjson.dumps( + { + "target_type": "artist", + "direction": "backward", + "type": link.link.link_type.name, + "type_id": str(link.link.link_type.gid), + "artist": artist_dict, + } + ).decode("utf-8") + ) + for link in url.release_links: + release = link.release + release_dict = {"id": str(release.gid), "title": release.name} + if release.comment: + release_dict["disambiguation"] = release.comment + relations.append( + orjson.dumps( + { + "target_type": "release", + "direction": "backward", + "type": link.link.link_type.name, + "type_id": str(link.link.link_type.gid), + "release": release_dict, + } + ).decode("utf-8") + ) + return relations + + +def work_relations_to_json(work): + relations = [] + for link in work.artist_links: + artist = link.artist + artist_dict = {"id": str(artist.gid), "name": artist.name} + if artist.comment: + artist_dict["disambiguation"] = artist.comment + if artist.sort_name is not None: + artist_dict["sort_name"] = artist.sort_name + relations.append( + orjson.dumps( + { + "target_type": "artist", + "direction": "backward", + "type": link.link.link_type.name, + "type_id": str(link.link.link_type.gid), + "artist": artist_dict, + } + ).decode("utf-8") + ) + for link in work.recording_links: + recording = link.recording + recording_dict = {"id": str(recording.gid), "title": recording.name} + if recording.video: + recording_dict["video"] = True + relations.append( + orjson.dumps( + { + "target_type": "recording", + "direction": "backward", + "type": link.link.link_type.name, + "type_id": str(link.link.link_type.gid), + "recording": recording_dict, + } + ).decode("utf-8") + ) + return relations + + +def event_relations_to_json(event): + relations = [] + for link in event.area_links: + area = link.area + relations.append( + orjson.dumps( + { + "target_type": "area", + "type": link.link.link_type.name, + "direction": "backward", + "type_id": str(link.link.link_type.gid), + "area": {"id": str(area.gid), "name": area.name}, + } + ).decode("utf-8") + ) + for link in event.artist_links: + artist = link.artist + artist_dict = {"id": str(artist.gid), "name": artist.name} + if artist.comment: + artist_dict["disambiguation"] = artist.comment + if artist.sort_name is not None: + artist_dict["sort_name"] = artist.sort_name + relations.append( + orjson.dumps( + { + "target_type": "artist", + "direction": "backward", + "type": link.link.link_type.name, + "type_id": str(link.link.link_type.gid), + "artist": artist_dict, + } + ).decode("utf-8") + ) + for link in event.place_links: + place = link.place + relations.append( + orjson.dumps( + { + "target_type": "place", + "direction": "backward", + "type": link.link.link_type.name, + "type_id": str(link.link.link_type.gid), + "place": {"id": str(place.gid), "name": place.name}, + } + ).decode("utf-8") + ) + return relations + + +def _release_event_dict(country_date): + area = country_date.country.area + return { + "area": { + "id": str(area.gid), + "name": area.name, + "iso_3166_1_codes": [c.code for c in area.iso_3166_1_codes], + }, + "date": partialdate_to_string(country_date.date), + } + + +def _medium_from_track_dict(track): + medium = track.medium + medium_dict = {"id": str(medium.gid)} + if medium.format is not None: + medium_dict["format"] = medium.format.name + medium_dict["track_count"] = medium.track_count + + medium_dict["position"] = medium.position + medium_dict["track_offset"] = track.position - 1 + + track_dict = {"id": str(track.gid)} + if track.length is not None: + track_dict["length"] = track.length + track_dict["number"] = track.number + track_dict["title"] = track.name + + medium_dict["track"] = track_dict + return medium_dict + + +def _release_group_for_release_dict(release_group): + release_group_dict = {"id": str(release_group.gid), "title": release_group.name} + if release_group.type is not None: + release_group_dict["primarytype"] = release_group.type.name + release_group_dict["primarytype_gid"] = str(release_group.type.gid) + calc_type = calculate_type(release_group.type, release_group.secondary_types) + release_group_dict["calc_type"] = calc_type.name + release_group_dict["calc_type_gid"] = str(calc_type.gid) + if release_group.secondary_types: + release_group_dict["secondarytype"] = [ + t.secondary_type.name for t in release_group.secondary_types + ] + release_group_dict["secondarytype_gid"] = [ + str(t.secondary_type.gid) for t in release_group.secondary_types + ] + if release_group.comment: + release_group_dict["disambiguation"] = release_group.comment + return release_group_dict + +@lru_cache(maxsize=5000) +def _artist_credit_dict(artist_credit, include_aliases): + name_credits = [] + for nc in artist_credit.artists: + artist = nc.artist + artist_dict = {"id": str(artist.gid), "name": artist.name} + if artist.comment: + artist_dict["disambiguation"] = artist.comment + if artist.sort_name is not None: + artist_dict["sort_name"] = artist.sort_name + + if include_aliases and artist.aliases: + artist_dict["aliases"] = [_alias_dict(a) for a in artist.aliases] + + credit = {"name": nc.name, "artist": artist_dict} + if nc.join_phrase != "": + credit["joinphrase"] = nc.join_phrase + name_credits.append(credit) + + return {"id": str(artist_credit.gid), "name_credits": name_credits} + + +def recording_artist_credit_to_json(recording): + return orjson.dumps(_artist_credit_dict(recording.artist_credit, True)).decode("utf-8") + + +def recording_releases_to_json(recording): + recording_credit = _artist_credit_dict(recording.artist_credit, True) + releases = [] + for track in recording.tracks: + release = track.medium.release + release_dict = {"id": str(release.gid), "title": release.name} + + release_credit = _artist_credit_dict(release.artist_credit, False) + if release_credit != recording_credit: + release_dict["artist_credit"] = release_credit + + if release.comment: + release_dict["disambiguation"] = release.comment + + if release.country_dates: + release_dict["release_events"] = [ + _release_event_dict(cd) for cd in release.country_dates + ] + + release_dict["medium"] = _medium_from_track_dict(track) + release_dict["medium_count"] = len(release.mediums) + release_dict["medium_track_count"] = sum(int(m.track_count) for m in release.mediums) + + release_dict["release_group"] = _release_group_for_release_dict(release.release_group) + + if release.status is not None: + release_dict["status"] = release.status.name + release_dict["status_id"] = str(release.status.gid) + + releases.append(orjson.dumps(release_dict).decode("utf-8")) + return releases + + +def release_artist_credit_to_json(release): + return orjson.dumps(_artist_credit_dict(release.artist_credit, False)).decode("utf-8") + + +def _release_event_dict(country_date): + area = country_date.country.area + return { + "area": { + "id": str(area.gid), + "name": area.name, + "iso_3166_1_codes": [c.code for c in area.iso_3166_1_codes], + }, + "date": partialdate_to_string(country_date.date), + } + + +def release_events_to_json(release): + return [orjson.dumps(_release_event_dict(cd)).decode("utf-8") for cd in release.country_dates] + + +def release_label_info_to_json(release): + label_infos = [] + for release_label in release.labels: + label_info = {} + if release_label.catalog_number: + label_info["catalog_number"] = release_label.catalog_number + if release_label.label is not None: + label_info["label"] = { + "id": str(release_label.label.gid), + "name": release_label.label.name, + } + label_infos.append(orjson.dumps(label_info).decode("utf-8")) + return label_infos + + +def release_mediums_to_json(release): + mediums = [] + for medium in release.mediums: + medium_dict = {"id": str(medium.gid)} + if medium.format is not None: + medium_dict["format"] = medium.format.name + medium_dict["disc_count"] = len(medium.cdtocs) + medium_dict["track_count"] = medium.track_count + mediums.append(orjson.dumps(medium_dict).decode("utf-8")) + return mediums + + +def release_barcode_none(release): + return "true" if release.barcode == "" else None + + +def release_calc_type(release): + return release_group_calc_type(release.release_group) + + +def release_calc_type_gid(release): + return release_group_calc_type_gid(release.release_group) + + +def release_group_calc_type(release_group): + if release_group.type is None: + return None + return calculate_type(release_group.type, release_group.secondary_types).name + + +def release_group_calc_type_gid(release_group): + if release_group.type is None: + return None + return str(calculate_type(release_group.type, release_group.secondary_types).gid) + + +def release_group_artist_credit_to_json(release_group): + return orjson.dumps( + _artist_credit_dict(release_group.artist_credit, True) + ).decode("utf-8") + + +def release_group_releases_to_json(release_group): + releases = [] + for release in release_group.releases: + release_dict = {"id": str(release.gid), "title": release.name} + if release.status is not None: + release_dict["status"] = release.status.name + release_dict["status_id"] = str(release.status.gid) + releases.append(orjson.dumps(release_dict).decode("utf-8")) + return releases + + def qdur(durations): if len(durations): return durations.pop() // 2000 @@ -109,10 +497,14 @@ def url_type(values): return types -def uuid_set_to_str_set(values): - return {str(x) for x in values} +def uuid_list_to_str_list(values): + return [str(x) for x in values] def datetime_to_timestamp(values): if values: return int(values.pop().timestamp()) + + +def str_list_to_lowercase(values: list[str]) -> list: + return [string.lower() for string in values] diff --git a/test/models.py b/test/models.py index a1767ef8..71af5b5a 100644 --- a/test/models.py +++ b/test/models.py @@ -1,5 +1,5 @@ from collections import namedtuple -from sqlalchemy import Column, ForeignKey, Integer, String +from sqlalchemy import Column, ForeignKey, Integer, PickleType, String from sqlalchemy.orm import composite, relationship, declarative_base Base = declarative_base() @@ -16,6 +16,7 @@ class B(Base): __tablename__ = "table_b" id = Column(Integer, primary_key=True) foo = Column(Integer) + tag_count = Column(PickleType) c_id = Column('c', Integer, ForeignKey("table_c.id")) composite_column = composite(Comp, foo, c_id) diff --git a/test/sql/release.sql b/test/sql/release.sql index 7bb7ad97..85eec28e 100644 --- a/test/sql/release.sql +++ b/test/sql/release.sql @@ -45,6 +45,8 @@ INSERT INTO release (id, gid, name, artist_credit, release_group) VALUES (3, '25 ; INSERT INTO medium (id, gid, track_count, release, position) VALUES (1, 'c517968f-afd0-48e6-ab4b-dfdae888ad9d', 1, 3, 1); +INSERT INTO medium (id, gid, track_count, release, position) VALUES (2, 'e517968f-afd0-48e6-ab4b-dfdae888ad9d', 3, 3, 2); + INSERT INTO recording (id, artist_credit, name, gid) VALUES (1, 2, 'Track on recording', 'b43eb990-ff5b-11de-8a39-0800200c9a66'); INSERT INTO track (id, gid, name, artist_credit, medium, position, number, recording) diff --git a/test/test_indexing_real_data.py b/test/test_indexing_real_data.py index 437cc6c2..0122e434 100644 --- a/test/test_indexing_real_data.py +++ b/test/test_indexing_real_data.py @@ -2,12 +2,11 @@ import unittest from pprint import pprint from queue import Queue -from datetime import datetime, timezone from sqlalchemy import text from sqlalchemy.orm import Session -from sir import querying, util, config +from sir import config, querying, util from sir.indexing import index_entity from sir.schema import SCHEMA @@ -66,132 +65,158 @@ def _test_index_entity(self, entity, expected_messages, key="mbid"): def test_index_area(self): expected = [ { - '_store': 'EuropeEuropeXEfalse', - 'area': u'Europe', - 'iso1': u'XE', - 'ended': 'false', - 'mbid': '89a675c2-3e37-3518-b83c-418bad59a85a', - 'type': u'Country' - }, - { - '_store': 'United StatesUnited StatesUSfalse', - 'area': u'United States', - 'iso1': u'US', - 'ended': 'false', - 'mbid': '489ce91b-6658-3307-9877-795b68554c98', - 'type': u'Country' - }, - { - '_store': 'United KingdomUnited KingdomGBfalse', - 'area': u'United Kingdom', - 'iso1': u'GB', - 'ended': 'false', - 'mbid': '8a754a16-0027-3a29-b6d7-2b40ea0481ed', - 'type': u'Country' - }, - { - '_store': 'JapanJapanJPfalse', - 'area': u'Japan', - 'iso1': u'JP', - 'ended': 'false', - 'mbid': '2db42837-c832-3c27-b4a3-08198f75693c', - 'type': u'Country' - }, - { - '_store': 'GermanyGermanyDEfalse', - 'area': u'Germany', - 'iso1': u'DE', - 'ended': 'false', - 'mbid': '85752fda-13c4-31a3-bee5-0e5cb1f51dad', - 'type': u'Country' - }, - { - '_store': 'AustraliaAustraliaAUfalseオーストラリア', - 'sortname': u'\u30aa\u30fc\u30b9\u30c8\u30e9\u30ea\u30a2', - 'ended': 'false', - 'area': u'Australia', - 'iso1': u'AU', - 'alias': u'\u30aa\u30fc\u30b9\u30c8\u30e9\u30ea\u30a2', - 'mbid': '106e0bec-b638-3b37-b731-f53d507dc00e', - 'type': u'Country' - }, - { - '_store': 'SydneySydneyfalse106e0bec-b638-3b37-b731-f53d507dc00ebackwardAustraliaAustraliafalse', - 'ended': 'false', - 'mbid': '3f179da4-83c6-4a28-a627-e46b4a8ff1ed', - 'type': u'City', - 'area': u'Sydney' - } + "area": "Europe", + "ended": "false", + "iso1": "XE", + "mbid": "89a675c2-3e37-3518-b83c-418bad59a85a", + "ref_count": 0, + "type": "Country", + "type_gid": "06dd0ae4-8c74-30bb-b43d-95dcedf961de", + }, + { + "area": "United States", + "ended": "false", + "iso1": "US", + "mbid": "489ce91b-6658-3307-9877-795b68554c98", + "ref_count": 0, + "type": "Country", + "type_gid": "06dd0ae4-8c74-30bb-b43d-95dcedf961de", + }, + { + "area": "United Kingdom", + "ended": "false", + "iso1": "GB", + "mbid": "8a754a16-0027-3a29-b6d7-2b40ea0481ed", + "ref_count": 0, + "type": "Country", + "type_gid": "06dd0ae4-8c74-30bb-b43d-95dcedf961de", + }, + { + "area": "Japan", + "ended": "false", + "iso1": "JP", + "mbid": "2db42837-c832-3c27-b4a3-08198f75693c", + "ref_count": 0, + "type": "Country", + "type_gid": "06dd0ae4-8c74-30bb-b43d-95dcedf961de", + }, + { + "area": "Germany", + "ended": "false", + "iso1": "DE", + "mbid": "85752fda-13c4-31a3-bee5-0e5cb1f51dad", + "ref_count": 0, + "type": "Country", + "type_gid": "06dd0ae4-8c74-30bb-b43d-95dcedf961de", + }, + { + "alias": "オーストラリア", + "alias_json": ['{"name":"オーストラリア","sort_name":"オーストラリア"}'], + "area": "Australia", + "ended": "false", + "iso1": "AU", + "mbid": "106e0bec-b638-3b37-b731-f53d507dc00e", + "ref_count": 0, + "sortname": "オーストラリア", + "type": "Country", + "type_gid": "06dd0ae4-8c74-30bb-b43d-95dcedf961de", + }, + { + "area": "Sydney", + "ended": "false", + "mbid": "3f179da4-83c6-4a28-a627-e46b4a8ff1ed", + "ref_count": 0, + "rel_json": [ + ( + '{"direction":"backward","type":"part ' + 'of","type_id":"de7cc874-8b1b-3a05-8272-f3834c968fb7","target":"106e0bec-b638-3b37-b731-f53d507dc00e","area":{"id":"106e0bec-b638-3b37-b731-f53d507dc00e","name":"Australia","type":"Country","type_id":"06dd0ae4-8c74-30bb-b43d-95dcedf961de","ended":"false"}}' + ) + ], + "type": "City", + "type_gid": "6fd8f29a-3d0a-32fc-980d-ea697b69da78", + }, ] self._test_index_entity("area", expected) def test_index_artist(self): expected = [ { - 'comment': u'Yet Another Test Artist', - 'begin': '2008-01-02', - 'endarea': u'United Kingdom', - 'end': '2009-03-04', - 'sortname': u'Artist, Test', - 'artist': u'Test Artist', - 'country': u'GB', - 'area': u'United Kingdom', - 'ended': 'true', - 'mbid': '745c079d-374e-4436-9448-da92dedef3ce', - 'gender': u'Male', - '_store': 'Test ArtistArtist, TestmaleGBUnited KingdomUnited KingdomfalseUnited KingdomUnited KingdomfalseUnited KingdomUnited KingdomfalseYet Another Test Artist2008-01-022009-03-04true', - 'type': u'Person', - 'beginarea': u'United Kingdom' - }, - { - 'ended': 'false', - 'mbid': 'ca4c2228-227c-4904-932a-dff442c091ea', - '_store': 'Annotated Artist BAnnotated Artist Bfalse', - 'sortname': u'Annotated Artist B', - 'artist': u'Annotated Artist B' - }, - { - 'ended': 'false', - 'mbid': 'dc19b13a-5ca5-44f5-8f0e-0c37a8ab1958', - '_store': 'Annotated Artist AAnnotated Artist Afalse', - 'sortname': u'Annotated Artist A', - 'artist': u'Annotated Artist A' - }, - { - 'ended': 'false', - 'mbid': '945c079d-374e-4436-9448-da92dedef3cf', - '_store': 'Minimal ArtistMinimal Artistfalse', - 'sortname': u'Minimal Artist', - 'artist': u'Minimal Artist' - } + "area": "United Kingdom", + "area_ended": "false", + "area_gid": "8a754a16-0027-3a29-b6d7-2b40ea0481ed", + "area_name": "United Kingdom", + "area_type": "Country", + "area_type_gid": "06dd0ae4-8c74-30bb-b43d-95dcedf961de", + "artist": "Test Artist", + "begin": "2008-01-02", + "beginarea": "United Kingdom", + "beginarea_ended": "false", + "beginarea_gid": "8a754a16-0027-3a29-b6d7-2b40ea0481ed", + "beginarea_name": "United Kingdom", + "beginarea_type": "Country", + "beginarea_type_gid": "06dd0ae4-8c74-30bb-b43d-95dcedf961de", + "comment": "Yet Another Test Artist", + "country": "GB", + "end": "2009-03-04", + "endarea": "United Kingdom", + "endarea_ended": "false", + "endarea_gid": "8a754a16-0027-3a29-b6d7-2b40ea0481ed", + "endarea_name": "United Kingdom", + "endarea_type": "Country", + "endarea_type_gid": "06dd0ae4-8c74-30bb-b43d-95dcedf961de", + "ended": "true", + "gender_gid": "36d3d30a-839d-3eda-8cb3-29be4384e4a9", + "gender_name": "male", + "mbid": "745c079d-374e-4436-9448-da92dedef3ce", + "ref_count": 0, + "sortname": "Artist, Test", + "type": "Person", + "type_gid": "b6e035f4-3ce9-331c-97df-83397230b0df", + }, + { + "area_ended": "false", + "artist": "Minimal Artist", + "beginarea_ended": "false", + "endarea_ended": "false", + "ended": "false", + "mbid": "945c079d-374e-4436-9448-da92dedef3cf", + "ref_count": 0, + "sortname": "Minimal Artist", + }, + { + "area_ended": "false", + "artist": "Annotated Artist A", + "beginarea_ended": "false", + "endarea_ended": "false", + "ended": "false", + "mbid": "dc19b13a-5ca5-44f5-8f0e-0c37a8ab1958", + "ref_count": 0, + "sortname": "Annotated Artist A", + }, + { + "area_ended": "false", + "artist": "Annotated Artist B", + "beginarea_ended": "false", + "endarea_ended": "false", + "ended": "false", + "mbid": "ca4c2228-227c-4904-932a-dff442c091ea", + "ref_count": 0, + "sortname": "Annotated Artist B", + }, ] self._test_index_entity("artist", expected) def test_index_editor(self): expected = [ { - 'bio': u'ModBot is a bot used by the MusicBrainz Server to perform a variety of automated functions. \\r+', - '_store': 'ModBotModBot is a bot used by the MusicBrainz Server to perform a variety of automated functions. \\r+', - 'id': 4, 'editor': u'ModBot' - }, - { - 'bio': u'biography', - '_store': 'new_editorbiography', - 'id': 1, - 'editor': u'new_editor' - }, - { - 'bio': u'second biography', - '_store': 'Alicesecond biography', - 'id': 2, - 'editor': u'Alice' + "bio": "ModBot is a bot used by the MusicBrainz Server to perform a variety of automated " + "functions. \\r+", + "editor": "ModBot", + "id": 4, }, - { - 'bio': u'donation check test user', - '_store': 'kunodonation check test user', - 'id': 3, - 'editor': u'kuno' - } + {"bio": "biography", "editor": "new_editor", "id": 1}, + {"bio": "second biography", "editor": "Alice", "id": 2}, + {"bio": "donation check test user", "editor": "kuno", "id": 3}, ] self._test_index_entity("editor", expected, key="id") @@ -199,82 +224,100 @@ def test_index_instrument(self): # Klavier/piano is present in the test database by default so account for that expected = [ { - '_store': 'pianoKlavier', - 'alias': 'Klavier', - 'instrument': 'piano', - 'mbid': 'b3eac5f9-7859-4416-ac39-7154e2e8d348', - 'type': 'String instrument' - }, - { - 'comment': u'Yet Another Test Instrument', - '_store': 'Test InstrumentYet Another Test InstrumentThis is a description!', - 'description': u'This is a description!', - 'instrument': u'Test Instrument', - 'mbid': '745c079d-374e-4436-9448-da92dedef3ce', - 'type': u'String instrument' + "comment": "Yet Another Test Instrument", + "description": "This is a description!", + "instrument": "Test Instrument", + "mbid": "745c079d-374e-4436-9448-da92dedef3ce", + "type": "String instrument", + "type_gid": "cc00f97f-cf3d-3ae2-9163-041cb1a0d726", }, { - 'instrument': u'Minimal Instrument 2', - 'mbid': 'a56d18ae-485f-5547-a559-eba3efef04d0', - '_store': 'Minimal Instrument 2' + "alias": "Klavier", + "alias_json": [ + ( + '{"name":"Klavier","sort_name":"Klavier","locale":"de","primary":"primary","type":"Instrument ' + 'name","type_id":"2322fc94-fbf3-3c09-b23c-aa5ec8d14fcd"}' + ) + ], + "instrument": "piano", + "mbid": "b3eac5f9-7859-4416-ac39-7154e2e8d348", + "type": "String instrument", + "type_gid": "cc00f97f-cf3d-3ae2-9163-041cb1a0d726", }, - { - 'instrument': u'Minimal Instrument', - 'mbid': '945c079d-374e-4436-9448-da92dedef3cf', - '_store': 'Minimal Instrument' - } + {"instrument": "Minimal Instrument 2", "mbid": "a56d18ae-485f-5547-a559-eba3efef04d0"}, + {"instrument": "Minimal Instrument", "mbid": "945c079d-374e-4436-9448-da92dedef3cf"}, ] self._test_index_entity("instrument", expected) def test_index_label(self): expected = [ { - 'comment': u'Sheffield based electronica label', - 'begin': '1989-02-03', - 'code': 2070, - 'end': '2008-05-19', - 'area': u'United Kingdom', - 'country': u'GB', - 'label': u'Warp Records', - 'ended': 'true', - 'mbid': '46f0f4cd-8aab-4b33-b698-f459faf64190', - '_store': 'Warp RecordsWarp Records2070Sheffield based electronica labelGBUnited KingdomUnited Kingdomfalse1989-02-032008-05-19true', - 'type': u'Production' - }, - { - 'ended': 'false', - 'mbid': 'f2a9a3c0-72e3-11de-8a39-0800200c9a66', - '_store': 'To MergeTo Mergefalse', - 'label': u'To Merge' - }, - { - 'begin': '1953-03-15', - 'end': '1991-11-27', - 'area': u'Soviet Union', - 'country': u'SU', - 'label': u'U.S.S.R. Ministry of Culture', - 'ended': 'true', - 'mbid': '449ddb7e-4e92-41eb-a683-5bbcc7fd7d4a', - '_store': 'U.S.S.R. Ministry of CultureU.S.S.R. Ministry of CultureSUSoviet UnionSoviet Union19221991true1953-03-151991-11-27true' - } + "area": "United Kingdom", + "area_ended": "false", + "area_gid": "8a754a16-0027-3a29-b6d7-2b40ea0481ed", + "area_name": "United Kingdom", + "area_type": "Country", + "area_type_gid": "06dd0ae4-8c74-30bb-b43d-95dcedf961de", + "begin": "1989-02-03", + "code": 2070, + "comment": "Sheffield based electronica label", + "country": "GB", + "end": "2008-05-19", + "ended": "true", + "label": "Warp Records", + "mbid": "46f0f4cd-8aab-4b33-b698-f459faf64190", + "release_count": 0, + "type": "Production", + "type_gid": "a2426aab-2dd4-339c-b47d-b4923a241678", + }, + { + "area_ended": "false", + "ended": "false", + "label": "To Merge", + "mbid": "f2a9a3c0-72e3-11de-8a39-0800200c9a66", + "release_count": 0, + }, + { + "area": "Soviet Union", + "area_begindate": "1922", + "area_enddate": "1991", + "area_ended": "true", + "area_gid": "32f90933-b4b4-3248-b98c-e573d5329f57", + "area_name": "Soviet Union", + "area_type": "Country", + "area_type_gid": "06dd0ae4-8c74-30bb-b43d-95dcedf961de", + "begin": "1953-03-15", + "country": "SU", + "end": "1991-11-27", + "ended": "true", + "label": "U.S.S.R. Ministry of Culture", + "mbid": "449ddb7e-4e92-41eb-a683-5bbcc7fd7d4a", + "release_count": 0, + }, ] self._test_index_entity("label", expected) def test_index_place(self): expected = [ { - 'comment': u'A PLACE!', - 'begin': '2013', - '_store': 'A Test PlaceA PLACE!An Address0.3231.234EuropeEuropefalse2013falseA Test Alias', - 'area': u'Europe', - 'long': 1.234, - 'alias': u'A Test Alias', - 'mbid': 'df9269dd-0470-4ea2-97e8-c11e46080edd', - 'ended': 'false', - 'address': u'An Address', - 'lat': 0.323, - 'place': u'A Test Place', - 'type': u'Venue' + "address": "An Address", + "alias": "A Test Alias", + "alias_json": ['{"name":"A Test Alias","sort_name":"A Test Alias"}'], + "area": "Europe", + "area_ended": "false", + "area_gid": "89a675c2-3e37-3518-b83c-418bad59a85a", + "area_name": "Europe", + "area_type": "Country", + "area_type_gid": "06dd0ae4-8c74-30bb-b43d-95dcedf961de", + "begin": "2013", + "comment": "A PLACE!", + "ended": "false", + "lat": 0.323, + "long": 1.234, + "mbid": "df9269dd-0470-4ea2-97e8-c11e46080edd", + "place": "A Test Place", + "type": "Venue", + "type_gid": "cd92781a-a73f-30e8-a430-55d7521338db", } ] self._test_index_entity("place", expected) @@ -282,388 +325,474 @@ def test_index_place(self): def test_index_recording(self): expected = [ { - 'primarytype': u'Album', - 'firstreleasedate': '2007', - '_store': 'King of the MountainArtistArtistArtist2007AerialAerialAlbum51Format1King of the Mountain', - 'tracks': 5, - 'format': u'Format', - 'creditname': u'Artist', - 'reid': 'f205627f-b70a-409d-adbe-66289b614e80', - 'artist': u'Artist', - 'mbid': '54b9d183-7dab-42ba-94a3-7388a66604b8', - 'arid': '945c079d-374e-4436-9448-da92dedef3cf', - 'number': u'1', - 'recording': u'King of the Mountain', - 'tid': '66c2ebff-86a8-4e12-a9a2-1650fb97d9d8', - 'artistname': u'Artist', - 'video': 'f', - 'rgid': '7c3218d7-75e0-4e8c-971f-f097b6c308c5', - 'tracksrelease': 5, - 'release': u'Aerial', - 'position': 1, - 'tnum': 1 - }, - { - '_store': 'Joanni296160ArtistArtistArtist', - 'qdur': 148, - 'artist': u'Artist', - 'creditname': u'Artist', - 'artistname': u'Artist', - 'arid': '945c079d-374e-4436-9448-da92dedef3cf', - 'recording': u'Joanni', - 'mbid': '07614140-8bb8-4db9-9dcc-0917c3a8471b', - 'video': 'f', - 'dur': 296160 - }, - { - 'tnum': 5, - 'primarytype': u'Album', - '_store': 'How to Be Invisible332613ArtistArtistArtist2007AerialAerialAlbum51Format5How to Be Invisible332613', - 'qdur': 166, - 'number': u'5', - 'video': 'f', - 'recording': u'How to Be Invisible', - 'creditname': u'Artist', - 'arid': '945c079d-374e-4436-9448-da92dedef3cf', - 'tracksrelease': 5, - 'tid': '849dc232-c33a-4611-a6a5-5a0969d63422', - 'dur': 332613, - 'firstreleasedate': '2007', - 'format': u'Format', - 'rgid': '7c3218d7-75e0-4e8c-971f-f097b6c308c5', - 'artistname': u'Artist', 'artist': u'Artist', - 'tracks': 5, - 'reid': 'f205627f-b70a-409d-adbe-66289b614e80', - 'mbid': '44f52946-0c98-47ba-ba60-964774db56f0', - 'release': u'Aerial', - 'position': 1 - }, - { - 'tnum': 4, - 'primarytype': u'Album', - '_store': 'Mrs. Bartolozzi358960ArtistArtistArtist2007AerialAerialAlbum51Format4Mrs. Bartolozzi358960', - 'qdur': 179, - 'number': u'4', - 'video': 'f', - 'recording': u'Mrs. Bartolozzi', - 'creditname': u'Artist', - 'arid': '945c079d-374e-4436-9448-da92dedef3cf', - 'tracksrelease': 5, - 'tid': '6c04d03c-4995-43be-8530-215ca911dcbf', - 'dur': 358960, - 'firstreleasedate': '2007', - 'format': u'Format', - 'rgid': '7c3218d7-75e0-4e8c-971f-f097b6c308c5', - 'artistname': u'Artist', - 'artist': u'Artist', - 'tracks': 5, - 'reid': 'f205627f-b70a-409d-adbe-66289b614e80', - 'mbid': 'b1d58a57-a0f3-4db8-aa94-868cdc7bc3bb', - 'release': u'Aerial', - 'position': 1 - }, - { - 'tnum': 3, - 'primarytype': u'Album', - '_store': 'Bertie258839ArtistArtistArtist2007AerialAerialAlbum51Format3Bertie258839', - 'qdur': 129, - 'number': u'3', - 'video': 'f', - 'recording': u'Bertie', - 'creditname': u'Artist', - 'arid': '945c079d-374e-4436-9448-da92dedef3cf', - 'tracksrelease': 5, - 'tid': 'f891acda-39d6-4a7f-a9d1-dd87b7c46a0a', - 'dur': 258839, - 'firstreleasedate': '2007', - 'format': u'Format', - 'rgid': '7c3218d7-75e0-4e8c-971f-f097b6c308c5', - 'artistname': u'Artist', - 'artist': u'Artist', - 'tracks': 5, - 'reid': 'f205627f-b70a-409d-adbe-66289b614e80', - 'mbid': 'ae674299-2824-4500-9516-653ac1bc6f80', - 'release': u'Aerial', - 'position': 1 - }, - { - 'tnum': 2, - 'primarytype': u'Album', - '_store': 'π369680ArtistArtistArtist2007AerialAerialAlbum51Format2π369680', - 'qdur': 184, - 'number': u'2', - 'video': 'f', - 'recording': u'\u03c0', - 'creditname': u'Artist', - 'arid': '945c079d-374e-4436-9448-da92dedef3cf', - 'tracksrelease': 5, - 'tid': 'b0caa7d1-0d1e-483e-b22b-ec6ab7fada06', - 'dur': 369680, - 'firstreleasedate': '2007', - 'format': u'Format', - 'rgid': '7c3218d7-75e0-4e8c-971f-f097b6c308c5', - 'artistname': u'Artist', - 'artist': u'Artist', - 'tracks': 5, - 'reid': 'f205627f-b70a-409d-adbe-66289b614e80', - 'mbid': '659f405b-b4ee-4033-868a-0daa27784b89', - 'release': u'Aerial', - 'position': 1 - } + "arid": "945c079d-374e-4436-9448-da92dedef3cf", + "artist": "Artist", + "artist_credit_json": '{"id":"949a7fd5-fe73-3e8f-922e-01ff4ca958f7","name_credits":[{"name":"Artist","artist":{"id":"945c079d-374e-4436-9448-da92dedef3cf","name":"Artist","sort_name":"Artist"}}]}', + "artistname": "Artist", + "creditname": "Artist", + "firstreleasedate": "2007", + "format": "Format", + "mbid": "54b9d183-7dab-42ba-94a3-7388a66604b8", + "number": "1", + "position": 1, + "primarytype": "Album", + "recording": "King of the Mountain", + "reid": "f205627f-b70a-409d-adbe-66289b614e80", + "release": "Aerial", + "release_json": [ + ( + '{"id":"f205627f-b70a-409d-adbe-66289b614e80","title":"Aerial","medium":{"id":"6e8ede88-4145-4412-8951-9e5ba757ea29","format":"Format","track_count":5,"position":1,"track_offset":0,"track":{"id":"66c2ebff-86a8-4e12-a9a2-1650fb97d9d8","number":"1","title":"King ' + "of the " + 'Mountain"}},"medium_count":2,"medium_track_count":5,"release_group":{"id":"7c3218d7-75e0-4e8c-971f-f097b6c308c5","title":"Aerial","primarytype":"Album","primarytype_gid":"f529b476-6e62-324f-b0aa-1f3e33d313fc","calc_type":"Album","calc_type_gid":"f529b476-6e62-324f-b0aa-1f3e33d313fc"}}' + ) + ], + "rgid": "7c3218d7-75e0-4e8c-971f-f097b6c308c5", + "tid": "66c2ebff-86a8-4e12-a9a2-1650fb97d9d8", + "tnum": 1, + "tracks": 5, + "tracksrelease": 5, + "video": "f", + }, + { + "arid": "945c079d-374e-4436-9448-da92dedef3cf", + "artist": "Artist", + "artist_credit_json": '{"id":"949a7fd5-fe73-3e8f-922e-01ff4ca958f7","name_credits":[{"name":"Artist","artist":{"id":"945c079d-374e-4436-9448-da92dedef3cf","name":"Artist","sort_name":"Artist"}}]}', + "artistname": "Artist", + "creditname": "Artist", + "dur": 296160, + "mbid": "07614140-8bb8-4db9-9dcc-0917c3a8471b", + "qdur": 148, + "recording": "Joanni", + "tracksrelease": 0, + "video": "f", + }, + { + "arid": "945c079d-374e-4436-9448-da92dedef3cf", + "artist": "Artist", + "artist_credit_json": '{"id":"949a7fd5-fe73-3e8f-922e-01ff4ca958f7","name_credits":[{"name":"Artist","artist":{"id":"945c079d-374e-4436-9448-da92dedef3cf","name":"Artist","sort_name":"Artist"}}]}', + "artistname": "Artist", + "creditname": "Artist", + "dur": 332613, + "firstreleasedate": "2007", + "format": "Format", + "mbid": "44f52946-0c98-47ba-ba60-964774db56f0", + "number": "5", + "position": 1, + "primarytype": "Album", + "qdur": 166, + "recording": "How to Be Invisible", + "reid": "f205627f-b70a-409d-adbe-66289b614e80", + "release": "Aerial", + "release_json": [ + ( + '{"id":"f205627f-b70a-409d-adbe-66289b614e80","title":"Aerial","medium":{"id":"6e8ede88-4145-4412-8951-9e5ba757ea29","format":"Format","track_count":5,"position":1,"track_offset":4,"track":{"id":"849dc232-c33a-4611-a6a5-5a0969d63422","length":332613,"number":"5","title":"How ' + "to Be " + 'Invisible"}},"medium_count":2,"medium_track_count":5,"release_group":{"id":"7c3218d7-75e0-4e8c-971f-f097b6c308c5","title":"Aerial","primarytype":"Album","primarytype_gid":"f529b476-6e62-324f-b0aa-1f3e33d313fc","calc_type":"Album","calc_type_gid":"f529b476-6e62-324f-b0aa-1f3e33d313fc"}}' + ) + ], + "rgid": "7c3218d7-75e0-4e8c-971f-f097b6c308c5", + "tid": "849dc232-c33a-4611-a6a5-5a0969d63422", + "tnum": 5, + "tracks": 5, + "tracksrelease": 5, + "video": "f", + }, + { + "arid": "945c079d-374e-4436-9448-da92dedef3cf", + "artist": "Artist", + "artist_credit_json": '{"id":"949a7fd5-fe73-3e8f-922e-01ff4ca958f7","name_credits":[{"name":"Artist","artist":{"id":"945c079d-374e-4436-9448-da92dedef3cf","name":"Artist","sort_name":"Artist"}}]}', + "artistname": "Artist", + "creditname": "Artist", + "dur": 358960, + "firstreleasedate": "2007", + "format": "Format", + "mbid": "b1d58a57-a0f3-4db8-aa94-868cdc7bc3bb", + "number": "4", + "position": 1, + "primarytype": "Album", + "qdur": 179, + "recording": "Mrs. Bartolozzi", + "reid": "f205627f-b70a-409d-adbe-66289b614e80", + "release": "Aerial", + "release_json": [ + ( + '{"id":"f205627f-b70a-409d-adbe-66289b614e80","title":"Aerial","medium":{"id":"6e8ede88-4145-4412-8951-9e5ba757ea29","format":"Format","track_count":5,"position":1,"track_offset":3,"track":{"id":"6c04d03c-4995-43be-8530-215ca911dcbf","length":358960,"number":"4","title":"Mrs. ' + 'Bartolozzi"}},"medium_count":2,"medium_track_count":5,"release_group":{"id":"7c3218d7-75e0-4e8c-971f-f097b6c308c5","title":"Aerial","primarytype":"Album","primarytype_gid":"f529b476-6e62-324f-b0aa-1f3e33d313fc","calc_type":"Album","calc_type_gid":"f529b476-6e62-324f-b0aa-1f3e33d313fc"}}' + ) + ], + "rgid": "7c3218d7-75e0-4e8c-971f-f097b6c308c5", + "tid": "6c04d03c-4995-43be-8530-215ca911dcbf", + "tnum": 4, + "tracks": 5, + "tracksrelease": 5, + "video": "f", + }, + { + "arid": "945c079d-374e-4436-9448-da92dedef3cf", + "artist": "Artist", + "artist_credit_json": '{"id":"949a7fd5-fe73-3e8f-922e-01ff4ca958f7","name_credits":[{"name":"Artist","artist":{"id":"945c079d-374e-4436-9448-da92dedef3cf","name":"Artist","sort_name":"Artist"}}]}', + "artistname": "Artist", + "creditname": "Artist", + "dur": 258839, + "firstreleasedate": "2007", + "format": "Format", + "mbid": "ae674299-2824-4500-9516-653ac1bc6f80", + "number": "3", + "position": 1, + "primarytype": "Album", + "qdur": 129, + "recording": "Bertie", + "reid": "f205627f-b70a-409d-adbe-66289b614e80", + "release": "Aerial", + "release_json": [ + '{"id":"f205627f-b70a-409d-adbe-66289b614e80","title":"Aerial","medium":{"id":"6e8ede88-4145-4412-8951-9e5ba757ea29","format":"Format","track_count":5,"position":1,"track_offset":2,"track":{"id":"f891acda-39d6-4a7f-a9d1-dd87b7c46a0a","length":258839,"number":"3","title":"Bertie"}},"medium_count":2,"medium_track_count":5,"release_group":{"id":"7c3218d7-75e0-4e8c-971f-f097b6c308c5","title":"Aerial","primarytype":"Album","primarytype_gid":"f529b476-6e62-324f-b0aa-1f3e33d313fc","calc_type":"Album","calc_type_gid":"f529b476-6e62-324f-b0aa-1f3e33d313fc"}}' + ], + "rgid": "7c3218d7-75e0-4e8c-971f-f097b6c308c5", + "tid": "f891acda-39d6-4a7f-a9d1-dd87b7c46a0a", + "tnum": 3, + "tracks": 5, + "tracksrelease": 5, + "video": "f", + }, + { + "arid": "945c079d-374e-4436-9448-da92dedef3cf", + "artist": "Artist", + "artist_credit_json": '{"id":"949a7fd5-fe73-3e8f-922e-01ff4ca958f7","name_credits":[{"name":"Artist","artist":{"id":"945c079d-374e-4436-9448-da92dedef3cf","name":"Artist","sort_name":"Artist"}}]}', + "artistname": "Artist", + "creditname": "Artist", + "dur": 369680, + "firstreleasedate": "2007", + "format": "Format", + "mbid": "659f405b-b4ee-4033-868a-0daa27784b89", + "number": "2", + "position": 1, + "primarytype": "Album", + "qdur": 184, + "recording": "π", + "reid": "f205627f-b70a-409d-adbe-66289b614e80", + "release": "Aerial", + "release_json": [ + '{"id":"f205627f-b70a-409d-adbe-66289b614e80","title":"Aerial","medium":{"id":"6e8ede88-4145-4412-8951-9e5ba757ea29","format":"Format","track_count":5,"position":1,"track_offset":1,"track":{"id":"b0caa7d1-0d1e-483e-b22b-ec6ab7fada06","length":369680,"number":"2","title":"π"}},"medium_count":2,"medium_track_count":5,"release_group":{"id":"7c3218d7-75e0-4e8c-971f-f097b6c308c5","title":"Aerial","primarytype":"Album","primarytype_gid":"f529b476-6e62-324f-b0aa-1f3e33d313fc","calc_type":"Album","calc_type_gid":"f529b476-6e62-324f-b0aa-1f3e33d313fc"}}' + ], + "rgid": "7c3218d7-75e0-4e8c-971f-f097b6c308c5", + "tid": "b0caa7d1-0d1e-483e-b22b-ec6ab7fada06", + "tnum": 2, + "tracks": 5, + "tracksrelease": 5, + "video": "f", + }, ] self._test_index_entity("recording", expected) def test_index_release(self): expected = [ { - 'primarytype': u'Album', - '_store': 'Release #2NameNameNameArrivalCommentAlbum', - 'artist': u'Name', - 'creditname': u'Name', - 'artistname': u'Name', - 'arid': 'a9d99e40-72d7-11de-8a39-0800200c9a66', - 'mbid': '7a906020-72db-11de-8a39-0800200c9a66', - 'rgid': '3b4faa80-72d9-11de-8a39-0800200c9a66', - 'release': u'Release #2', - 'quality': -1 - }, - { - 'comment': u'Comment', - 'lang': u'deu', - 'script': u'Ugar', - '_store': 'ArrivalOfficialCommentJewel CasedeuUgarNameNameNameArrivalCommentAlbum2009-05-08GB2009-05-08United KingdomUnited KingdomGB731453398122ABC-123-XLabelABC-123Label', - 'artist': u'Name', - 'creditname': u'Name', - 'country': u'GB', - 'barcode': u'731453398122', - 'status': u'Official', - 'artistname': u'Name', - 'arid': 'a9d99e40-72d7-11de-8a39-0800200c9a66', - 'label': u'Label', - 'packaging': u'Jewel Case', - 'date': '2009-05-08', - 'mbid': 'f34c079d-374e-4436-9448-da92dedef3ce', - 'catno': [u'ABC-123', u'ABC-123-X'], - 'rgid': '3b4faa80-72d9-11de-8a39-0800200c9a66', - 'laid': '00a23bd0-72db-11de-8a39-0800200c9a66', - 'release': u'Arrival', - 'quality': -1, - 'primarytype': u'Album' - }, - { - '_store': 'Blonde on BlondeVarious ArtistsVarious ArtistsVarious ArtistsBlonde on Blonde', - 'artist': u'Various Artists', - 'creditname': u'Various Artists', - 'artistname': u'Various Artists', - 'arid': '7a906020-72db-11de-8a39-0800200c9a66', - 'mbid': '538aff00-a009-4515-a064-11a6d5a502ee', - 'rgid': '329fb554-2a81-3d8a-8e22-ec2c66810019', - 'release': u'Blonde on Blonde', - 'quality': -1}, - { - '_store': 'Various ReleaseVarious ArtistsVarious ArtistsVarious ArtistsVarious Release3', - 'tracks': 3, 'artist': u'Various Artists', - 'creditname': u'Various Artists', - 'artistname': u'Various Artists', - 'arid': '7a906020-72db-11de-8a39-0800200c9a66', - 'tracksmedium': 3, - 'mbid': '25b6fe30-ff5b-11de-8a39-0800200c9a66', - 'mediumid': 'c517968f-afd0-48e6-ab4b-dfdae888ad9d', - 'rgid': '25b6fe30-ff5b-11de-8a39-0800200c9a66', - 'release': u'Various Release', - 'mediums': 1, - 'quality': -1 - } + "arid": "a9d99e40-72d7-11de-8a39-0800200c9a66", + "artist": "Name", + "artist_credit_json": '{"id":"949a7fd5-fe73-3e8f-922e-01ff4ca958f7","name_credits":[{"name":"Name","artist":{"id":"a9d99e40-72d7-11de-8a39-0800200c9a66","name":"Name","sort_name":"Name"}}]}', + "artistname": "Name", + "calc_type": "Album", + "calc_type_gid": "f529b476-6e62-324f-b0aa-1f3e33d313fc", + "creditname": "Name", + "mbid": "7a906020-72db-11de-8a39-0800200c9a66", + "mediums": 0, + "primarytype": "Album", + "primarytype_gid": "f529b476-6e62-324f-b0aa-1f3e33d313fc", + "quality": -1, + "release": "Release #2", + "rg_comment": "Comment", + "rg_name": "Arrival", + "rgid": "3b4faa80-72d9-11de-8a39-0800200c9a66", + "tracks": 0, + }, + { + "arid": "a9d99e40-72d7-11de-8a39-0800200c9a66", + "artist": "Name", + "artist_credit_json": '{"id":"949a7fd5-fe73-3e8f-922e-01ff4ca958f7","name_credits":[{"name":"Name","artist":{"id":"a9d99e40-72d7-11de-8a39-0800200c9a66","name":"Name","sort_name":"Name"}}]}', + "artistname": "Name", + "barcode": "731453398122", + "calc_type": "Album", + "calc_type_gid": "f529b476-6e62-324f-b0aa-1f3e33d313fc", + "catno": ["ABC-123-X", "ABC-123"], + "comment": "Comment", + "country": "GB", + "creditname": "Name", + "date": "2009-05-08", + "label": "Label", + "label_info_json": [ + '{"catalog_number":"ABC-123-X","label":{"id":"00a23bd0-72db-11de-8a39-0800200c9a66","name":"Label"}}', + '{"catalog_number":"ABC-123","label":{"id":"00a23bd0-72db-11de-8a39-0800200c9a66","name":"Label"}}', + ], + "laid": "00a23bd0-72db-11de-8a39-0800200c9a66", + "lang": "deu", + "mbid": "f34c079d-374e-4436-9448-da92dedef3ce", + "mediums": 0, + "packaging": "Jewel Case", + "packaging_gid": "ec27701a-4a22-37f4-bfac-6616e0f9750a", + "primarytype": "Album", + "primarytype_gid": "f529b476-6e62-324f-b0aa-1f3e33d313fc", + "quality": -1, + "release": "Arrival", + "release_event_json": [ + ( + '{"area":{"id":"8a754a16-0027-3a29-b6d7-2b40ea0481ed","name":"United ' + 'Kingdom","iso_3166_1_codes":["GB"]},"date":"2009-05-08"}' + ) + ], + "rg_comment": "Comment", + "rg_name": "Arrival", + "rgid": "3b4faa80-72d9-11de-8a39-0800200c9a66", + "script": "Ugar", + "status": "Official", + "status_gid": "4e304316-386d-3409-af2e-78857eec5cfe", + "tracks": 0, + }, + { + "arid": "7a906020-72db-11de-8a39-0800200c9a66", + "artist": "Various Artists", + "artist_credit_json": '{"id":"c44109ce-57d7-3691-84c8-37926e3d41d2","name_credits":[{"name":"Various ' + 'Artists","artist":{"id":"7a906020-72db-11de-8a39-0800200c9a66","name":"Various ' + 'Artists","sort_name":"Various Artists"}}]}', + "artistname": "Various Artists", + "creditname": "Various Artists", + "mbid": "538aff00-a009-4515-a064-11a6d5a502ee", + "mediums": 0, + "quality": -1, + "release": "Blonde on Blonde", + "rg_name": "Blonde on Blonde", + "rgid": "329fb554-2a81-3d8a-8e22-ec2c66810019", + "tracks": 0, + }, + { + "arid": "7a906020-72db-11de-8a39-0800200c9a66", + "artist": "Various Artists", + "artist_credit_json": '{"id":"c44109ce-57d7-3691-84c8-37926e3d41d2","name_credits":[{"name":"Various ' + 'Artists","artist":{"id":"7a906020-72db-11de-8a39-0800200c9a66","name":"Various ' + 'Artists","sort_name":"Various Artists"}}]}', + "artistname": "Various Artists", + "creditname": "Various Artists", + "mbid": "25b6fe30-ff5b-11de-8a39-0800200c9a66", + "medium_json": [ + '{"id":"c517968f-afd0-48e6-ab4b-dfdae888ad9d","disc_count":0,"track_count":3}', + '{"id":"e517968f-afd0-48e6-ab4b-dfdae888ad9d","disc_count":0,"track_count":3}', + ], + "mediumid": [ + "e517968f-afd0-48e6-ab4b-dfdae888ad9d", + "c517968f-afd0-48e6-ab4b-dfdae888ad9d", + ], + "mediums": 2, + "quality": -1, + "release": "Various Release", + "rg_name": "Various Release", + "rgid": "25b6fe30-ff5b-11de-8a39-0800200c9a66", + "tracks": 6, + "tracksmedium": [3, 3], + }, ] self._test_index_entity("release", expected) def test_index_release_group(self): expected = [ { - 'reid': '25b6fe30-ff5b-11de-8a39-0800200c9a66', - '_store': 'Various ReleaseVarious ArtistsVarious ArtistsVarious ArtistsVarious Release', - 'releases': 1, - 'artist': u'Various Artists', - 'creditname': u'Various Artists', - 'artistname': u'Various Artists', - 'arid': '7a906020-72db-11de-8a39-0800200c9a66', - 'releasegroup': u'Various Release', - 'mbid': '25b6fe30-ff5b-11de-8a39-0800200c9a66', - 'release': u'Various Release' - }, - { - 'comment': u'Comment', - 'reid': '4c767e70-72d8-11de-8a39-0800200c9a66', - '_store': 'Release GroupCommentAlbumNameName1Release Name', - 'releases': 1, - 'artist': u'Name', - 'creditname': u'Name', - 'primarytype': u'Album', - 'artistname': u'Name', - 'arid': 'a9d99e40-72d7-11de-8a39-0800200c9a66', - 'releasegroup': u'Release Group', - 'mbid': '7b5d22d0-72d7-11de-8a39-0800200c9a66', - 'release': u'Release Name' - }, - { - 'comment': u'Comment', - 'primarytype': u'Album', - '_store': 'Release NameCommentAlbumNameName1', - 'artist': u'Name', - 'creditname': u'Name', - 'artistname': u'Name', - 'arid': 'a9d99e40-72d7-11de-8a39-0800200c9a66', - 'releasegroup': u'Release Name', - 'mbid': '3b4faa80-72d9-11de-8a39-0800200c9a66' - } + "arid": "7a906020-72db-11de-8a39-0800200c9a66", + "artist": "Various Artists", + "artist_credit_json": '{"id":"c44109ce-57d7-3691-84c8-37926e3d41d2","name_credits":[{"name":"Various ' + 'Artists","artist":{"id":"7a906020-72db-11de-8a39-0800200c9a66","name":"Various ' + 'Artists","sort_name":"Various Artists"}}]}', + "artistname": "Various Artists", + "creditname": "Various Artists", + "mbid": "25b6fe30-ff5b-11de-8a39-0800200c9a66", + "reid": "25b6fe30-ff5b-11de-8a39-0800200c9a66", + "release": "Various Release", + "release_json": [ + '{"id":"25b6fe30-ff5b-11de-8a39-0800200c9a66","title":"Various Release"}' + ], + "releasegroup": "Various Release", + "releases": 1, + }, + { + "arid": "a9d99e40-72d7-11de-8a39-0800200c9a66", + "artist": "Name", + "artist_credit_json": '{"id":"949a7fd5-fe73-3e8f-922e-01ff4ca958f7","name_credits":[{"name":"Name","artist":{"id":"a9d99e40-72d7-11de-8a39-0800200c9a66","name":"Name","sort_name":"1"}}]}', + "artistname": "Name", + "calc_type": "Album", + "calc_type_gid": "f529b476-6e62-324f-b0aa-1f3e33d313fc", + "comment": "Comment", + "creditname": "Name", + "mbid": "7b5d22d0-72d7-11de-8a39-0800200c9a66", + "primarytype": "Album", + "primarytype_gid": "f529b476-6e62-324f-b0aa-1f3e33d313fc", + "reid": "4c767e70-72d8-11de-8a39-0800200c9a66", + "release": "Release Name", + "release_json": [ + '{"id":"4c767e70-72d8-11de-8a39-0800200c9a66","title":"Release Name"}' + ], + "releasegroup": "Release Group", + "releases": 1, + }, + { + "arid": "a9d99e40-72d7-11de-8a39-0800200c9a66", + "artist": "Name", + "artist_credit_json": '{"id":"949a7fd5-fe73-3e8f-922e-01ff4ca958f7","name_credits":[{"name":"Name","artist":{"id":"a9d99e40-72d7-11de-8a39-0800200c9a66","name":"Name","sort_name":"1"}}]}', + "artistname": "Name", + "calc_type": "Album", + "calc_type_gid": "f529b476-6e62-324f-b0aa-1f3e33d313fc", + "comment": "Comment", + "creditname": "Name", + "mbid": "3b4faa80-72d9-11de-8a39-0800200c9a66", + "primarytype": "Album", + "primarytype_gid": "f529b476-6e62-324f-b0aa-1f3e33d313fc", + "releasegroup": "Release Name", + "releases": 0, + }, ] self._test_index_entity("release-group", expected) def test_index_series(self): expected = [ { - 'series': u'Dumb Recording Series', - 'mbid': 'dbb23c50-d4e4-11e3-9c1a-0800200c9a66', - 'type': u'Recording series', - '_store': 'Dumb Recording Series' + "mbid": "dbb23c50-d4e4-11e3-9c1a-0800200c9a66", + "series": "Dumb Recording Series", + "type": "Recording series", + "type_gid": "dd968243-7128-30a2-81f0-79843430a8e2", }, { - 'comment': u'test comment 1', - '_store': 'Test Recording Seriestest comment 1Test Recording Series Alias', - 'series': u'Test Recording Series', - 'alias': u'Test Recording Series Alias', - 'mbid': 'a8749d0c-4a5a-4403-97c5-f6cd018f8e6d', - 'type': u'Recording series', + "alias": "Test Recording Series Alias", + "alias_json": [ + ( + '{"name":"Test Recording Series Alias","sort_name":"Test Recording Series ' + 'Alias","type":"Search hint","type_id":"8950366b-5ea3-32f2-bf74-ee482474c18b"}' + ) + ], + "comment": "test comment 1", + "mbid": "a8749d0c-4a5a-4403-97c5-f6cd018f8e6d", + "series": "Test Recording Series", + "type": "Recording series", + "type_gid": "dd968243-7128-30a2-81f0-79843430a8e2", }, { - 'comment': u'test comment 2', - 'series': u'Test Work Series', - 'mbid': '2e8872b9-2745-4807-a84e-094d425ec267', - 'type': u'Work series', - '_store': 'Test Work Seriestest comment 2' - } + "comment": "test comment 2", + "mbid": "2e8872b9-2745-4807-a84e-094d425ec267", + "series": "Test Work Series", + "type": "Work series", + "type_gid": "b689f694-6305-3d78-954d-df6759a1877b", + }, ] self._test_index_entity("series", expected) def test_index_tag(self): expected = [ - { - 'tag': u'musical', - '_store': 'musical', - 'id': 1 - }, - { - 'tag': u'rock', - '_store': 'rock', - 'id': 2 - }, - { - 'tag': u'jazz', - '_store': 'jazz', - 'id': 3 - }, - { - 'tag': u'world music', - '_store': 'world music', - 'id': 4 - } + {"id": 1, "tag": "musical"}, + {"id": 2, "tag": "rock"}, + {"id": 3, "tag": "jazz"}, + {"id": 4, "tag": "world music"}, ] self._test_index_entity("tag", expected, key="id") def test_index_url(self): expected = [ - { - 'url': u'http://musicbrainz.org/', - 'mbid': '9201840b-d810-4e0f-bb75-c791205f5b24', - '_store': 'http://musicbrainz.org/' - }, - { - 'url': u'http://microsoft.com', - 'mbid': '9b3c5c67-572a-4822-82a3-bdd3f35cf152', - '_store': 'http://microsoft.com' - }, - { - 'targettype': 'artist', - '_store': 'http://zh-yue.wikipedia.org/wiki/%E7%8E%8B%E8%8F%B2backwardFaye WongFaye Wong', - 'url': u'http://zh-yue.wikipedia.org/wiki/%E7%8E%8B%E8%8F%B2', - 'targetid': 'acd58926-4243-40bb-a2e5-c7464b3ce577', - 'mbid': '25d6b63a-12dc-41c9-858a-2f42ae610a7d', - 'relationtype': u'wikipedia' - }, - { - 'targettype': 'artist', - '_store': 'https://www.allmusic.com/artist/faye-wong-mn0000515659backwardFaye WongFaye Wong', - 'url': u'https://www.allmusic.com/artist/faye-wong-mn0000515659', - 'targetid': 'acd58926-4243-40bb-a2e5-c7464b3ce577', - 'mbid': '7bd45cc7-6189-4712-35e1-cdf3632cf1a9', - 'relationtype': u'allmusic' + {"mbid": "9201840b-d810-4e0f-bb75-c791205f5b24", "url": "http://musicbrainz.org/"}, + {"mbid": "9b3c5c67-572a-4822-82a3-bdd3f35cf152", "url": "http://microsoft.com"}, + { + "mbid": "25d6b63a-12dc-41c9-858a-2f42ae610a7d", + "rel_json": [ + ( + '{"target_type":"artist","direction":"backward","type":"wikipedia","type_id":"29651736-fa6d-48e4-aadc-a557c6add1cb","artist":{"id":"acd58926-4243-40bb-a2e5-c7464b3ce577","name":"Faye ' + 'Wong","sort_name":"Faye Wong"}}' + ) + ], + "relationtype": "wikipedia", + "targetid": "acd58926-4243-40bb-a2e5-c7464b3ce577", + "targettype": "artist", + "url": "http://zh-yue.wikipedia.org/wiki/%E7%8E%8B%E8%8F%B2", + }, + { + "mbid": "7bd45cc7-6189-4712-35e1-cdf3632cf1a9", + "rel_json": [ + ( + '{"target_type":"artist","direction":"backward","type":"allmusic","type_id":"6b3e3c85-0002-4f34-aca6-80ace0d7e846","artist":{"id":"acd58926-4243-40bb-a2e5-c7464b3ce577","name":"Faye ' + 'Wong","sort_name":"Faye Wong"}}' + ) + ], + "relationtype": "allmusic", + "targetid": "acd58926-4243-40bb-a2e5-c7464b3ce577", + "targettype": "artist", + "url": "https://www.allmusic.com/artist/faye-wong-mn0000515659", }, - { - 'url': u'http://microsoft.fr', - 'mbid': '9b3c5c67-572a-4822-82a3-bdd3f35cf153', - '_store': 'http://microsoft.fr' - } + {"mbid": "9b3c5c67-572a-4822-82a3-bdd3f35cf153", "url": "http://microsoft.fr"}, ] self._test_index_entity("url", expected) def test_index_work(self): expected = [ { - 'comment': u'Work', - '_store': 'TestWork', - 'mbid': '105c079d-374e-4436-9448-da92dedef3ce', - 'work': u'Test', - 'type': u'Aria' - }, - { - 'comment': u'Work', - '_store': 'TestT-500.000.001-0T-500.000.002-0Work', - 'iswc': [u'T-500.000.002-0', u'T-500.000.001-0'], - 'work': u'Test', - 'mbid': '755c079d-374e-4436-9448-da92dedef3ce', - 'type': u'Aria' - }, - { - 'comment': u'Work', - '_store': 'Dancing QueenT-000.000.001-0Work', - 'iswc': u'T-000.000.001-0', 'work': u'Dancing Queen', - 'mbid': '745c079d-374e-4436-9448-da92dedef3ce', - 'type': u'Aria' + "comment": "Work", + "mbid": "105c079d-374e-4436-9448-da92dedef3ce", + "recording_count": 0, + "type": "Aria", + "type_gid": "ae801f48-7a7f-3af6-91c7-456f82dae8a9", + "work": "Test", + }, + { + "comment": "Work", + "iswc": ["T-500.000.002-0", "T-500.000.001-0"], + "mbid": "755c079d-374e-4436-9448-da92dedef3ce", + "recording_count": 0, + "type": "Aria", + "type_gid": "ae801f48-7a7f-3af6-91c7-456f82dae8a9", + "work": "Test", + }, + { + "comment": "Work", + "iswc": "T-000.000.001-0", + "mbid": "745c079d-374e-4436-9448-da92dedef3ce", + "recording_count": 0, + "type": "Aria", + "type_gid": "ae801f48-7a7f-3af6-91c7-456f82dae8a9", + "work": "Dancing Queen", + }, + { + "mbid": "640b17f5-4aa3-3fb1-8c6c-4792458e8a56", + "recording": "Blue Lines", + "recording_count": 2, + "rel_json": [ + ( + '{"target_type":"recording","direction":"backward","type":"performance","type_id":"a3005666-a872-32c3-ad06-98af558e99b0","recording":{"id":"bef81f8f-4bcf-4308-bd66-e57018169a94","title":"Blue ' + 'Lines"}}' + ), + ( + '{"target_type":"recording","direction":"backward","type":"performance","type_id":"a3005666-a872-32c3-ad06-98af558e99b0","recording":{"id":"a2383c02-2430-4294-9177-ef799a6eca31","title":"Blue ' + 'Lines"}}' + ), + ], + "rid": [ + "bef81f8f-4bcf-4308-bd66-e57018169a94", + "a2383c02-2430-4294-9177-ef799a6eca31", + ], + "type": "Song", + "type_gid": "f061270a-2fd6-32f1-a641-f0f8676d14e6", + "work": "Blue Lines", }, { - 'mbid': '745c079d-374e-4436-9448-da92dedef3cf', - 'work': u'Test', - '_store': 'TestT-000.000.002-0', - 'iswc': u'T-000.000.002-0' + "iswc": "T-000.000.002-0", + "mbid": "745c079d-374e-4436-9448-da92dedef3cf", + "recording_count": 0, + "work": "Test", }, - { - '_store': 'Blue LinesbackwardBlue LinesbackwardBlue Lines', - 'work': u'Blue Lines', - 'recording_count': 2, - 'recording': u'Blue Lines', - 'mbid': '640b17f5-4aa3-3fb1-8c6c-4792458e8a56', - 'rid': ['bef81f8f-4bcf-4308-bd66-e57018169a94', 'a2383c02-2430-4294-9177-ef799a6eca31'], - 'type': u'Song' - } ] self._test_index_entity("work", expected) def test_index_cdstub(self): - expected_timestamp = int( - datetime(2000, 1, 1, tzinfo=timezone.utc) - .timestamp() - ) expected = [ { - 'comment': u'this is a comment', - 'added': expected_timestamp, - '_store': 'Test StubTest Artist837101029192this is a comment', - 'discid': u'YfSgiOEayqN77Irs.VNV.UNJ0Zs-', - 'artist': u'Test Artist', - 'barcode': u'837101029192', - 'tracks': 2, - 'title': u'Test Stub', - 'id': 1 + "added": 946684800, + "artist": "Test Artist", + "barcode": "837101029192", + "comment": "this is a comment", + "discid": "YfSgiOEayqN77Irs.VNV.UNJ0Zs-", + "id": 1, + "title": "Test Stub", + "tracks": 2, } ] self._test_index_entity("cdstub", expected, key="id") @@ -671,58 +800,70 @@ def test_index_cdstub(self): def test_index_annotation(self): expected = [ { - '_store': '745c079d-374e-4436-9448-da92dedef3ceTest ArtistTest annotation 1', - 'name': u'Test Artist', - 'text': u'Test annotation 1', - 'entity': '745c079d-374e-4436-9448-da92dedef3ce', - 'type': 'artist', - 'id': 1 + "entity": "745c079d-374e-4436-9448-da92dedef3ce", + "id": 1, + "name": "Test Artist", + "text": "Test annotation 1", + "type": "artist", }, { - '_store': '945c079d-374e-4436-9448-da92dedef3cfMinimal ArtistTest annotation 2', - 'name': u'Minimal Artist', - 'text': u'Test annotation 2', - 'entity': '945c079d-374e-4436-9448-da92dedef3cf', - 'type': 'artist', - 'id': 2 + "entity": "945c079d-374e-4436-9448-da92dedef3cf", + "id": 2, + "name": "Minimal Artist", + "text": "Test annotation 2", + "type": "artist", }, { - '_store': 'dc19b13a-5ca5-44f5-8f0e-0c37a8ab1958Annotated Artist ADuplicate annotation', - 'name': u'Annotated Artist A', - 'text': u'Duplicate annotation', - 'entity': 'dc19b13a-5ca5-44f5-8f0e-0c37a8ab1958', - 'type': 'artist', - 'id': 3 + "entity": "dc19b13a-5ca5-44f5-8f0e-0c37a8ab1958", + "id": 3, + "name": "Annotated Artist A", + "text": "Duplicate annotation", + "type": "artist", }, { - '_store': 'ca4c2228-227c-4904-932a-dff442c091eaAnnotated Artist BDuplicate annotation', - 'name': u'Annotated Artist B', - 'text': u'Duplicate annotation', - 'entity': 'ca4c2228-227c-4904-932a-dff442c091ea', - 'type': 'artist', - 'id': 4 - } + "entity": "ca4c2228-227c-4904-932a-dff442c091ea", + "id": 4, + "name": "Annotated Artist B", + "text": "Duplicate annotation", + "type": "artist", + }, ] self._test_index_entity("annotation", expected, key="id") def test_index_event(self): expected = [ { - 'comment': u'2022, Prom 60', - 'begin': '2022-09-01', - 'end': '2022-09-01', - 'artist': [u'Kwam\xe9 Ryan', u'BBC Concert Orchestra'], - 'pid': '4352063b-a833-421b-a420-e7fb295dece0', - 'arid': [ - 'f72a5b32-449f-4090-9a2a-ebbdd8d3c2e5', - 'dfeba5ea-c967-4ad2-9cdd-3cffb4320143' + "arid": [ + "dfeba5ea-c967-4ad2-9cdd-3cffb4320143", + "f72a5b32-449f-4090-9a2a-ebbdd8d3c2e5", + ], + "artist": ["BBC Concert Orchestra", "Kwamé Ryan"], + "begin": "2022-09-01", + "comment": "2022, Prom 60", + "end": "2022-09-01", + "ended": "true", + "event": "BBC Open Music Prom", + "mbid": "ca1d24c1-1999-46fd-8a95-3d4108df5cb2", + "pid": "4352063b-a833-421b-a420-e7fb295dece0", + "place": "Royal Albert Hall", + "rel_json": [ + ( + '{"target_type":"artist","direction":"backward","type":"orchestra","type_id":"9b2d5b96-b4d9-4bce-b056-c369ced25e81","artist":{"id":"dfeba5ea-c967-4ad2-9cdd-3cffb4320143","name":"BBC ' + 'Concert Orchestra","sort_name":"BBC Concert Orchestr"}}' + ), + ( + '{"target_type":"artist","direction":"backward","type":"conductor","type_id":"92873f0d-12a7-4fb3-9eac-ff06c38c6a60","artist":{"id":"f72a5b32-449f-4090-9a2a-ebbdd8d3c2e5","name":"Kwamé ' + 'Ryan","sort_name":"Ryan, Kwamé"}}' + ), + ( + '{"target_type":"place","direction":"backward","type":"held ' + 'at","type_id":"e2c6f697-07dc-38b1-be0b-83d740165532","place":{"id":"4352063b-a833-421b-a420-e7fb295dece0","name":"Royal ' + 'Albert Hall"}}' + ), ], - 'ended': 'true', - 'mbid': 'ca1d24c1-1999-46fd-8a95-3d4108df5cb2', - 'place': u'Royal Albert Hall', - '_store': 'BBC Open Music Prom2022, Prom 602022-09-012022-09-01true19:30:00backwardBBC Concert OrchestraBBC Concert OrchestrbackwardKwamé RyanRyan, KwamébackwardRoyal Albert Hall', - 'type': u'Concert', - 'event': u'BBC Open Music Prom' + "time": "19:30:00", + "type": "Concert", + "type_gid": "ef55e8d7-3d00-394a-8012-f5506a29ff0b", } ] self._test_index_entity("event", expected) diff --git a/test/test_searchentities.py b/test/test_searchentities.py index 58fa8352..ba90f4e3 100644 --- a/test/test_searchentities.py +++ b/test/test_searchentities.py @@ -1,12 +1,13 @@ -from unittest import mock, TestCase +from unittest import TestCase, mock -from test import models -from xml.etree.ElementTree import Element, tostring -from sir.schema.searchentities import (SearchEntity as E, SearchField as F, - is_composite_column) from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from sir.schema.searchentities import SearchEntity as E +from sir.schema.searchentities import SearchField as F +from sir.schema.searchentities import is_composite_column +from test import models + class QueryResultToDictTest(TestCase): def setUp(self): @@ -14,43 +15,51 @@ def setUp(self): self.addCleanup(config_patcher.stop) instance = config_patcher.start() instance.getboolean.return_value = True - self.entity = E(models.B, [ - F("id", "id"), - F("c_bar", "c.bar"), - F("c_bar_trans", "c.bar", transformfunc=lambda v: - v.union(set(["yay"]))) - ], - 1.1 + self.entity = E( + models.B, + [ + F("id", "id"), + F("foo", "foo"), + F("tag_count_unique", "tag_count"), + F("c_bar", "c.bar"), + F("c_bar_trans", "c.bar", transformfunc=lambda v: v + ["yay"]), + ], + 1.1, ) self.expected = { "id": 1, - "c_bar": "foo", - "c_bar_trans": ["yay", "foo"], + "foo": "foo", + "tag_count_unique": [1, 2, 0], + "c_bar": "bar", + "c_bar_trans": ["yay", "bar"], } - c = models.C(id=2, bar="foo") - self.val = models.B(id=1, c=c) + c = models.C(id=2, bar="bar") + self.val = models.B(id=1, c=c, tag_count=[1, 2, 0, 1], foo="foo") def test_fields(self): res = self.entity.query_result_to_dict(self.val) self.assertEqual(res.keys(), self.expected.keys()) self.assertEqual(res["id"], self.expected["id"]) + self.assertCountEqual(res["tag_count_unique"], self.expected["tag_count_unique"]) self.assertEqual(res["c_bar"], self.expected["c_bar"]) + self.assertEqual(res["foo"], self.expected["foo"]) self.assertCountEqual(res["c_bar_trans"], self.expected["c_bar_trans"]) - def test_conversion(self): - elem = Element("testelem", text="text") - convmock = mock.Mock() - convmock.to_etree.return_value = elem - self.entity.compatconverter = lambda x: convmock + def test_objconverter(self): + converter = mock.Mock(return_value=['{"c_bar": "bar", "foo": "foo"}']) + self.entity.fields.append(F("json", ["c.bar", "c.foo"], objconverter=converter)) res = self.entity.query_result_to_dict(self.val) - self.expected["_store"] = str(tostring(elem, encoding="us-ascii"), encoding="us-ascii") - self.assertEqual(res.keys(), self.expected.keys()) - self.assertEqual(res["id"], self.expected["id"]) - self.assertEqual(res["c_bar"], self.expected["c_bar"]) - self.assertCountEqual(res["c_bar_trans"], self.expected["c_bar_trans"]) - self.assertEqual(convmock.to_etree.call_count, 1) + self.assertEqual(res["json"], ['{"c_bar": "bar", "foo": "foo"}']) + converter.assert_called_once_with(self.val) + + def test_preserve_og(self): + self.entity.fields.append(F("tag_count", "tag_count", preserve_og=True)) + + result = self.entity.query_result_to_dict(self.val) + + self.assertEqual(result["tag_count"], [1, 2, 0, 1]) class TestIsCompositeColumn(TestCase):