fix(serializer): keep meta.language out of text exports and emit lang attributes in HTML

docling_core/transforms/serializer/html.py

@@ -117,7 +117,10 @@ class HTMLParams(CommonParams):
     image_mode: ImageRefMode = ImageRefMode.PLACEHOLDER
 
     # HTML document properties
-    html_lang: str = "en"
+    # Language of the page, emitted as ``<html lang>``. None resolves to the document
+    # language: ``doc.body.meta.language``, else the character-weighted majority of the
+    # text items' ``meta.language``, else "en".
+    html_lang: str | None = None
     html_head: str | None = None
 
     css_styles: str | None = None
@@ -143,6 +146,42 @@ class HTMLParams(CommonParams):
     show_original_list_item_marker: bool = True
 
 
+_DEFAULT_HTML_LANG = "en"
+
+
+def resolve_document_language(doc: DoclingDocument) -> str:
+    """Return the language to declare on ``<html lang>`` for the document.
+
+    The document-level language (``doc.body.meta.language``) wins. Without it, the language
+    that covers most characters across the text items that carry ``meta.language`` is used.
+    Without any language metadata the result is ``"en"``.
+    """
+    if doc.body.meta is not None and doc.body.meta.language is not None:
+        return doc.body.meta.language.code.value
+    weights: dict[str, int] = {}
+    for item in doc.texts:
+        if item.meta is not None and item.meta.language is not None:
+            code = item.meta.language.code.value
+            weights[code] = weights.get(code, 0) + len(item.text)
+    if not weights:
+        return _DEFAULT_HTML_LANG
+    return max(weights, key=lambda code: (weights[code], code))
+
+
+def _primary_subtag(lang: str) -> str:
+    return lang.split("-", 1)[0].lower()
+
+
+def _lang_attrs(item: NodeItem, page_lang: str | None) -> dict[str, str]:
+    """``lang`` attribute for an item whose language differs from the page language."""
+    if item.meta is None or item.meta.language is None:
+        return {}
+    code = item.meta.language.code.value
+    if page_lang is not None and _primary_subtag(page_lang) == _primary_subtag(code):
+        return {}
+    return {"lang": code}
+
+
 class HTMLTextSerializer(BaseModel, BaseTextSerializer):
     """HTML-specific text item serializer."""
 
@@ -162,6 +201,7 @@ def serialize(
         my_visited: set[str] = visited if visited is not None else set()
         res_parts: list[SerializationResult] = []
         post_processed = False
+        lang_attrs = _lang_attrs(item, params.html_lang)
 
         has_inline_repr = (
             item.text == ""
@@ -184,7 +224,7 @@ def serialize(
         # Prepare the HTML based on item type
         if isinstance(item, TitleItem | SectionHeaderItem):
             section_level = min(item.level + 1, 6) if isinstance(item, SectionHeaderItem) else 1
-            text = get_html_tag_with_text_direction(html_tag=f"h{section_level}", text=text)
+            text = get_html_tag_with_text_direction(html_tag=f"h{section_level}", text=text, attrs=lang_attrs)
 
         elif isinstance(item, FormulaItem):
             text = self._process_formula(
@@ -232,19 +272,22 @@ def serialize(
                 get_html_tag_with_text_direction(
                     html_tag="li",
                     text=text,
-                    attrs=(
-                        {"style": f"list-style-type: '{_escape_css_string(item.marker)} ';"}
-                        if params.show_original_list_item_marker and item.marker
-                        else {}
-                    ),
+                    attrs={
+                        **lang_attrs,
+                        **(
+                            {"style": f"list-style-type: '{_escape_css_string(item.marker)} ';"}
+                            if params.show_original_list_item_marker and item.marker
+                            else {}
+                        ),
+                    },
                 )
                 if text
                 else ""
             )
 
         elif not is_inline_scope:
             # Regular text item
-            text = get_html_tag_with_text_direction(html_tag="p", text=text)
+            text = get_html_tag_with_text_direction(html_tag="p", text=text, attrs=lang_attrs)
 
         # Apply formatting and hyperlinks to the parent's own text+tag.
         if not post_processed:
@@ -1131,6 +1174,12 @@ class HTMLDocSerializer(DocSerializer):
 
     hoist_decoration_whitespace: ClassVar[bool] = True
 
+    def model_post_init(self, __context: Any) -> None:
+        """Resolve the page language once so every element serializer sees the same value."""
+        super().model_post_init(__context)
+        if self.params.html_lang is None:
+            self.params = self.params.model_copy(update={"html_lang": resolve_document_language(self.doc)})
+
     @override
     def _item_wraps_meta(self, item: NodeItem) -> bool:
         """PictureItem meta is wrapped inside the figure element."""
@@ -1197,9 +1246,10 @@ def _serialize_page_img(page_img: Image):
             return f"<figure>{img_text}</figure>"
 
         # Create HTML structure
+        html_lang = self.params.html_lang or resolve_document_language(self.doc)
         html_parts = [
             "<!DOCTYPE html>",
-            "<html>",
+            f'<html lang="{html.escape(html_lang, quote=True)}">',
             self._generate_head(),
             "<body>",
         ]

docling_core/transforms/serializer/markdown.py

@@ -41,13 +41,15 @@
     BaseMeta,
     CaptionPlacement,
     CodeItem,
+    CodeMetaField,
     ContentLayer,
     CoordOrigin,
     DescriptionAnnotation,
     DescriptionMetaField,
     DocItem,
     DocItemLabel,
     DoclingDocument,
+    EntitiesMetaField,
     FieldItem,
     FieldRegionItem,
     FloatingItem,
@@ -60,6 +62,7 @@
     InlineGroup,
     KeyValueItem,
     KeywordsMetaField,
+    LanguageMetaField,
     ListGroup,
     ListItem,
     MoleculeMetaField,
@@ -230,10 +233,13 @@ def _code_span(text: str) -> str:
     return f"{ticks}{pad}{text}{pad}{ticks}"
 
 
-def _code_block(text: str) -> str:
-    """Wrap text in a fenced code block whose fence is longer than any backtick run inside."""
+def _code_block(text: str, lang: str = "") -> str:
+    """Wrap text in a fenced code block whose fence is longer than any backtick run inside.
+
+    ``lang`` becomes the info string of the opening fence.
+    """
     fence = "`" * max(3, _longest_backtick_run(text) + 1)
-    return f"{fence}\n{text}\n{fence}"
+    return f"{fence}{lang}\n{text}\n{fence}"
 
 
 class OrigListItemMarkerMode(str, Enum):
@@ -549,6 +555,7 @@ def serialize(
                                 key,
                                 params.mark_meta,
                                 include_picture_classification=params.include_picture_classification,
+                                format_code_blocks=params.format_code_blocks,
                             )
                         )
                     )
@@ -567,21 +574,40 @@ def _serialize_meta_field(
         mark_meta: bool,
         *,
         include_picture_classification: bool = True,
+        format_code_blocks: bool = True,
         **kwargs: Any,
     ) -> str | None:
         if (field_val := getattr(meta, name)) is not None:
+            fenced = False
             if isinstance(field_val, SummaryMetaField):
                 txt = field_val.text
             elif isinstance(field_val, KeywordsMetaField | TopicsMetaField):
                 txt = ", ".join(field_val.values)
             elif isinstance(field_val, DescriptionMetaField):
                 txt = field_val.text
+            elif isinstance(field_val, LanguageMetaField):
+                # A language code describes the text; it is not content. Only a marked
+                # export lists it.
+                if not mark_meta:
+                    return None
+                txt = field_val.code.value
+            elif isinstance(field_val, EntitiesMetaField):
+                if not mark_meta:
+                    return None
+                txt = ", ".join(mention.text for mention in field_val.mentions)
             elif isinstance(field_val, PictureClassificationMetaField):
                 if not include_picture_classification:
                     return None
                 txt = self._humanize_text(field_val.get_main_prediction().class_name)
             elif isinstance(field_val, MoleculeMetaField):
                 txt = field_val.smi
+            elif isinstance(field_val, CodeMetaField):
+                if format_code_blocks:
+                    lang = field_val.language.value.lower() if field_val.language else ""
+                    txt = _code_block(field_val.text, lang=lang)
+                    fenced = True
+                else:
+                    txt = field_val.text
             elif isinstance(field_val, TabularChartMetaField):
                 temp_doc = DoclingDocument(name="temp")
                 temp_table = temp_doc.add_table(data=field_val.chart_data)
@@ -590,11 +616,19 @@ def _serialize_meta_field(
                     txt = table_content
                 else:
                     return None
+            elif isinstance(field_val, BaseModel):
+                # A structured field with no renderer above: skip it rather than print its
+                # pydantic repr as a paragraph.
+                return None
             elif tmp := str(field_val or ""):
                 txt = tmp
             else:
                 return None
-            return f"[{self._humanize_text(name, title=True)}] {txt}" if mark_meta else txt
+            if not mark_meta:
+                return txt
+            label = f"[{self._humanize_text(name, title=True)}]"
+            # A code fence must start its own line, so the label goes on the line above it.
+            return f"{label}\n{txt}" if fenced else f"{label} {txt}"
         else:
             return None
 

docling_core/types/doc/document.py

@@ -4298,7 +4298,7 @@ def save_as_html(
         image_mode: ImageRefMode = ImageRefMode.PLACEHOLDER,
         formula_to_mathml: bool = True,
         page_no: int | None = None,
-        html_lang: str = "en",
+        html_lang: str | None = None,
         html_head: str = "null",  # should be deprecated
         included_content_layers: set[ContentLayer] | None = None,
         split_page_view: bool = False,
@@ -4396,7 +4396,7 @@ def export_to_html(
         image_mode: ImageRefMode = ImageRefMode.PLACEHOLDER,
         formula_to_mathml: bool = True,
         page_no: int | None = None,
-        html_lang: str = "en",
+        html_lang: str | None = None,
         html_head: str = "null",  # should be deprecated ...
         included_content_layers: set[ContentLayer] | None = None,
         split_page_view: bool = False,

tests/data/doc/2206.01062.yaml.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>2206.01062v1</title>

tests/data/doc/2408.09869_p1_split.gt.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>2408.09869_p1</title>

tests/data/doc/2408.09869v3_enriched_p1_include_annotations_false.gt.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head></head>
 <body>
 <div class='page'>

tests/data/doc/2408.09869v3_enriched_p1_include_annotations_true.gt.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head></head>
 <body>
 <div class='page'>

tests/data/doc/2408.09869v3_enriched_p2_p3_p5.gt.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>2408.09869v3</title>

tests/data/doc/2408.09869v3_enriched_split.gt.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>2408.09869v3</title>

tests/data/doc/2408.09869v3_enriched_split_p2.gt.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>2408.09869v3</title>

tests/data/doc/activities.gt.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>activities</title>

tests/data/doc/activities_p1.gt.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>activities</title>

tests/data/doc/activities_p2.gt.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>activities</title>

tests/data/doc/barchart.gt.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>Document</title>

tests/data/doc/concatenated.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>2501.17887v1 + Untitled 1 + 2311.18481v1</title>

tests/data/doc/constructed_doc.embedded.html.gt

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>Untitled 1</title>

tests/data/doc/constructed_doc.placeholder.html.gt

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>Untitled 1</title>

tests/data/doc/constructed_doc.referenced.html.gt

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>Untitled 1</title>

tests/data/doc/constructed_document.yaml.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>Untitled 1</title>

tests/data/doc/constructed_orig_false.gt.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>Untitled 1</title>

tests/data/doc/constructed_orig_true.gt.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>Untitled 1</title>

tests/data/doc/dummy_doc.yaml.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>dummy_doc</title>

tests/data/doc/inline_and_formatting.gt.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 <meta charset="UTF-8"/>
 <title>inline_and_formatting</title>