Skip to content

fix: read ToUnicode source codes as numbers, not UTF-16 text #419

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wittjeff wants to merge 1 commit into from
Closed

fix: read ToUnicode source codes as numbers, not UTF-16 text #419

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 22 additions & 58 deletions src/parse/pdf_resources/page_font/cmap.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ namespace pdflib
static std::string to_utf8(QPDFObjectHandle handle,
size_t number_of_chars);

std::string get_source(QPDFObjectHandle my_handle);
uint32_t get_source(QPDFObjectHandle my_handle);
std::string get_target(QPDFObjectHandle my_handle);

void parse_cmap_name(std::vector<qpdf_stream_instruction>& parameters);
Expand All @@ -44,15 +44,15 @@ namespace pdflib
void parse_beginbfchar(std::vector<qpdf_stream_instruction>& parameters);
void parse_endbfchar(std::vector<qpdf_stream_instruction>& parameters);

void set_mapping(const std::string src,
void set_mapping(uint32_t c,
const std::string tgt);

void set_range(const std::string src_begin,
const std::string src_end,
void set_range(uint32_t begin,
uint32_t end,
const std::string tgt);

void set_range(const std::string src_begin,
const std::string src_end,
void set_range(uint32_t begin,
uint32_t end,
const std::vector<std::string> tgt);

// A bfrange is expanded code by code, so a corrupt bound (srcCode2 below
Expand Down Expand Up @@ -677,19 +677,13 @@ namespace pdflib
LOG_S(INFO) << __FUNCTION__ << " bf_cnt: " << bf_cnt;
}

// the source can be 1 or 2 byte
std::string cmap_parser::get_source(QPDFObjectHandle my_handle)
// The source is a character code of 1 to 4 bytes (PDF 32000-1, 9.10.3),
// read as a big-endian number like the codespacerange bounds. It is not
// text: decoding it as UTF-16 rejected every code in 0xD800-0xDFFF, and
// the fallback string that came back put the mapping on the wrong code.
uint32_t cmap_parser::get_source(QPDFObjectHandle my_handle)
{
std::string source = my_handle.getStringValue();

std::string result="";
{
std::string tmp = my_handle.getStringValue();
result = to_utf8(my_handle, tmp.size());
}
//LOG_S(INFO) << __FUNCTION__ << my_handle.unparse() << "\t" << source << "\t'" << result << "'";

return result;
return to_uint32(my_handle);
}

// the target is always 2 byte
Expand Down Expand Up @@ -734,8 +728,8 @@ namespace pdflib
LOG_S(WARNING) << "parameters.size(): " << parameters.size() << " > " << num_params;
}

std::string source_start = "";
std::string source_end = "";
uint32_t source_start = 0;
uint32_t source_end = 0;

QPDFObjectHandle target;

Expand Down Expand Up @@ -799,16 +793,10 @@ namespace pdflib
}
}

void cmap_parser::set_mapping(const std::string src,
void cmap_parser::set_mapping(uint32_t c,
const std::string tgt)
{
//LOG_S(INFO) << __FUNCTION__ << ": " << src << " -> " << tgt;

uint32_t c = 0;
{
auto itr = src.begin();
c = utf8::next(itr, src.end());
}
//LOG_S(INFO) << __FUNCTION__ << ": " << c << " -> " << tgt;

if(not (csr_range.first<=c and c<=csr_range.second))
{
Expand All @@ -820,7 +808,7 @@ namespace pdflib
LOG_S(ERROR) << "overwriting number cmap[" << c << "]: " << _map.at(c) << " with " << tgt;
}

LOG_S(INFO) << "orig: " << src << " -> cmap-ind:" << c << " -> target: " << tgt;
LOG_S(INFO) << "cmap-ind:" << c << " -> target: " << tgt;
_map[c] = tgt;
}

Expand All @@ -837,30 +825,12 @@ namespace pdflib
return false;
}

void cmap_parser::set_range(const std::string src_begin,
const std::string src_end,
void cmap_parser::set_range(uint32_t begin,
uint32_t end,
const std::string tgt)
{
//LOG_S(INFO) << __FUNCTION__;

auto itr_beg = src_begin.begin();
uint32_t begin = utf8::next(itr_beg, src_begin.end());

if(itr_beg != src_begin.end())
{
LOG_S(WARNING) << "itr_beg!=src_begin.end() --> errors might occur in the cmap: "
<< "'" << src_begin << "' -> " << begin;
}

auto itr_end = src_end.begin();
uint32_t end = utf8::next(itr_end, src_end.end());

if(itr_end != src_end.end())
{
LOG_S(WARNING) << "itr_end!=src_end.end() --> errors might occur in the cmap: "
<< "'" << src_end << "' -> " << end;
}

if(not is_valid_range(begin, end))
{
return;
Expand Down Expand Up @@ -910,18 +880,12 @@ namespace pdflib
//populate_range_mapping_legacy(begin, end, mapping, tgts, csr_range, _map);
}

void cmap_parser::set_range(const std::string src_begin,
const std::string src_end,
void cmap_parser::set_range(uint32_t begin,
uint32_t end,
const std::vector<std::string> tgt)
{
//LOG_S(INFO) << __FUNCTION__;

auto itr_begin = src_begin.begin();
uint32_t begin = utf8::next(itr_begin, src_begin.end());

auto itr_end = src_end.begin();
uint32_t end = utf8::next(itr_end, src_end.end());

if(not is_valid_range(begin, end))
{
return;
Expand Down Expand Up @@ -999,7 +963,7 @@ namespace pdflib
QPDFObjectHandle source_ = parameters[2*i+0].obj;
QPDFObjectHandle target_ = parameters[2*i+1].obj;

std::string source = get_source(source_);
uint32_t source = get_source(source_);
std::string target = get_target(target_);

target.erase(std::remove_if(target.begin(), target.end(), [] (char x) { return x == 0; }),
Expand Down
54 changes: 54 additions & 0 deletions tests/test_unit_cmap_encoding.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,3 +186,57 @@ def test_reversed_bfrange_is_ignored():
assert text == "AB", (
f"expected the valid bfrange to map the codes to 'AB', got {text!r}"
)


TO_UNICODE_SURROGATE_RANGE_CODES = """/CIDInit /ProcSet findresource begin
12 dict begin
begincmap
/CMapName /Test-UCS2 def
/CMapType 2 def
1 begincodespacerange
<0000> <FFFF>
endcodespacerange
2 beginbfchar
<0047> <0058>
<DFA8> <AC00>
endbfchar
1 beginbfrange
<D800> <D801> <0041>
endbfrange
endcmap
CMapName currentdict /CMap defineresource pop
end
end"""


def test_bfchar_source_code_in_surrogate_range_is_mapped():
"""A source code between 0xD800 and 0xDFFF maps like any other code.

Source codes are byte sequences, not text. They were decoded as UTF-16,
which rejects lone surrogates; the fallback string that came back keyed
the mapping under 'G' (0x47). So the code itself lost its text, and code
0x47 took the text of the bad entry. Large CJK fonts reach this range: in
reported files every space of a Korean font came out as a quote mark.
"""
text = _text(
_identity_h_pdf(
text_bytes="DFA80047",
to_unicode=True,
to_unicode_cmap=TO_UNICODE_SURROGATE_RANGE_CODES,
)
)
assert text == "가X", (
f"expected <DFA8> to give U+AC00 and <0047> to keep 'X', got {text!r}"
)


def test_bfrange_bounds_in_surrogate_range_are_mapped():
"""A bfrange whose bounds lie between 0xD800 and 0xDFFF is expanded."""
text = _text(
_identity_h_pdf(
text_bytes="D800D801",
to_unicode=True,
to_unicode_cmap=TO_UNICODE_SURROGATE_RANGE_CODES,
)
)
assert text == "AB", f"expected the range to map to 'AB', got {text!r}"
Loading