Move ByteString::FromUnicode() to WideString::ToDefANSI() Turns out that "FromUnicode" is misleading in that, on linux, it simply removes any characters beyond 0xFF and passes the rest unchanged, so no unicode decoding actually takes place. On Windows, it passes it into the system function specifying FX_CODEPAGE_DefANSI, converting it into the so-called "default ANSI code plane", passing some characters, converting others to '?' and still others to 'A'. Either way, nothing resembling UTF8 comes out of this, so pick a better name. These now immediately look suspicious, so a follow-up CL will see which ones should really be WideString::UTF8Encode() instead. Making this a normal method on a widestring rather than a static method on a bytestring feels more natural; this is parallel to the UTF8Encode and UTF16LE_Encode functions. Add a test that shows these conversions. Change-Id: Ia7551b47199eba61b5c328a97bfe9176ac8e583c Reviewed-on: https://pdfium-review.googlesource.com/39690 Reviewed-by: Lei Zhang <thestig@chromium.org> Commit-Queue: Tom Sepez <tsepez@chromium.org>
diff --git a/core/fpdfapi/parser/cpdf_document.cpp b/core/fpdfapi/parser/cpdf_document.cpp index 7641dbd..2a43262 100644 --- a/core/fpdfapi/parser/cpdf_document.cpp +++ b/core/fpdfapi/parser/cpdf_document.cpp
@@ -814,7 +814,7 @@ bool bTranslateName) { LOGFONTA lfa; memcpy(&lfa, pLogFont, (char*)lfa.lfFaceName - (char*)&lfa); - ByteString face = ByteString::FromUnicode(pLogFont->lfFaceName); + ByteString face = WideString(pLogFont->lfFaceName).ToDefANSI(); if (face.GetLength() >= LF_FACESIZE) return nullptr;
diff --git a/core/fpdfdoc/cpdf_filespec.cpp b/core/fpdfdoc/cpdf_filespec.cpp index 2cc51e3..aa93b91 100644 --- a/core/fpdfdoc/cpdf_filespec.cpp +++ b/core/fpdfdoc/cpdf_filespec.cpp
@@ -203,10 +203,9 @@ WideString wsStr = EncodeFileName(wsFileName); if (m_pObj->IsString()) { - m_pWritableObj->SetString(ByteString::FromUnicode(wsStr)); + m_pWritableObj->SetString(wsStr.ToDefANSI()); } else if (CPDF_Dictionary* pDict = m_pWritableObj->AsDictionary()) { - pDict->SetNewFor<CPDF_String>(pdfium::stream::kF, - ByteString::FromUnicode(wsStr), false); + pDict->SetNewFor<CPDF_String>(pdfium::stream::kF, wsStr.ToDefANSI(), false); pDict->SetNewFor<CPDF_String>("UF", PDF_EncodeText(wsStr), false); } }
diff --git a/core/fpdfdoc/cpdf_interform.cpp b/core/fpdfdoc/cpdf_interform.cpp index d7cee35..26dfbc1 100644 --- a/core/fpdfdoc/cpdf_interform.cpp +++ b/core/fpdfdoc/cpdf_interform.cpp
@@ -1030,8 +1030,8 @@ if (!pdf_path.IsEmpty()) { if (bSimpleFileSpec) { WideString wsFilePath = CPDF_FileSpec::EncodeFileName(pdf_path); - pMainDict->SetNewFor<CPDF_String>( - pdfium::stream::kF, ByteString::FromUnicode(wsFilePath), false); + pMainDict->SetNewFor<CPDF_String>(pdfium::stream::kF, + wsFilePath.ToDefANSI(), false); pMainDict->SetNewFor<CPDF_String>("UF", PDF_EncodeText(wsFilePath), false); } else {
diff --git a/core/fxcrt/bytestring.cpp b/core/fxcrt/bytestring.cpp index 4d55c98..b6c1ce7 100644 --- a/core/fxcrt/bytestring.cpp +++ b/core/fxcrt/bytestring.cpp
@@ -676,26 +676,6 @@ return WideString(decoder.GetResult()); } -// static -ByteString ByteString::FromUnicode(const WideString& wstr) { - int src_len = wstr.GetLength(); - int dest_len = - FXSYS_WideCharToMultiByte(FX_CODEPAGE_DefANSI, 0, wstr.c_str(), src_len, - nullptr, 0, nullptr, nullptr); - if (!dest_len) - return ByteString(); - - ByteString bstr; - { - // Span's lifetime must end before ReleaseBuffer() below. - pdfium::span<char> dest_buf = bstr.GetBuffer(dest_len); - FXSYS_WideCharToMultiByte(FX_CODEPAGE_DefANSI, 0, wstr.c_str(), src_len, - dest_buf.data(), dest_len, nullptr, nullptr); - } - bstr.ReleaseBuffer(dest_len); - return bstr; -} - int ByteString::Compare(const ByteStringView& str) const { if (!m_pData) return str.IsEmpty() ? 0 : -1;
diff --git a/core/fxcrt/bytestring.h b/core/fxcrt/bytestring.h index 5722c49..5975acb 100644 --- a/core/fxcrt/bytestring.h +++ b/core/fxcrt/bytestring.h
@@ -66,8 +66,6 @@ void clear() { m_pData.Reset(); } - static ByteString FromUnicode(const WideString& str) WARN_UNUSED_RESULT; - // Explicit conversion to C-style string. // Note: Any subsequent modification of |this| will invalidate the result. const char* c_str() const { return m_pData ? m_pData->m_String : ""; }
diff --git a/core/fxcrt/widestring.cpp b/core/fxcrt/widestring.cpp index cde1973..7dd1c30 100644 --- a/core/fxcrt/widestring.cpp +++ b/core/fxcrt/widestring.cpp
@@ -673,6 +673,25 @@ return m_pData ? m_pData->m_nRefs : 0; } +// static +ByteString WideString::ToDefANSI() const { + int src_len = GetLength(); + int dest_len = FXSYS_WideCharToMultiByte( + FX_CODEPAGE_DefANSI, 0, c_str(), src_len, nullptr, 0, nullptr, nullptr); + if (!dest_len) + return ByteString(); + + ByteString bstr; + { + // Span's lifetime must end before ReleaseBuffer() below. + pdfium::span<char> dest_buf = bstr.GetBuffer(dest_len); + FXSYS_WideCharToMultiByte(FX_CODEPAGE_DefANSI, 0, c_str(), src_len, + dest_buf.data(), dest_len, nullptr, nullptr); + } + bstr.ReleaseBuffer(dest_len); + return bstr; +} + ByteString WideString::UTF8Encode() const { return FX_UTF8Encode(AsStringView()); }
diff --git a/core/fxcrt/widestring.h b/core/fxcrt/widestring.h index b531292..dc5dd23 100644 --- a/core/fxcrt/widestring.h +++ b/core/fxcrt/widestring.h
@@ -195,6 +195,7 @@ size_t Replace(const WideStringView& pOld, const WideStringView& pNew); size_t Remove(wchar_t ch); + ByteString ToDefANSI() const; ByteString UTF8Encode() const; // This method will add \0\0 to the end of the string to represent the
diff --git a/core/fxcrt/widestring_unittest.cpp b/core/fxcrt/widestring_unittest.cpp index 9017fe0..9d38aa45 100644 --- a/core/fxcrt/widestring_unittest.cpp +++ b/core/fxcrt/widestring_unittest.cpp
@@ -999,6 +999,32 @@ } } +TEST(WideString, ToDefANSI) { + EXPECT_EQ("", WideString().ToDefANSI()); +#if _FX_PLATFORM_ == _FX_PLATFORM_WINDOWS_ + const char* kResult = + "x" + "?" + "\xff" + "A" + "?" + "y"; +#else + const char* kResult = + "x" + "\x80" + "\xff" + "y"; +#endif + EXPECT_EQ(kResult, WideString(L"x" + L"\u0080" + L"\u00ff" + L"\u0100" + L"\u208c" + L"y") + .ToDefANSI()); +} + TEST(WideStringView, FromVector) { std::vector<WideStringView::UnsignedType> null_vec; WideStringView null_string(null_vec);
diff --git a/core/fxge/win32/fx_win32_print.cpp b/core/fxge/win32/fx_win32_print.cpp index f36fa843..1e2b2d4 100644 --- a/core/fxge/win32/fx_win32_print.cpp +++ b/core/fxge/win32/fx_win32_print.cpp
@@ -650,7 +650,7 @@ wsText += charpos.m_Unicode; } size_t len = totalLength; - ByteString text = ByteString::FromUnicode(wsText); + ByteString text = wsText.ToDefANSI(); while (len > 0) { char buffer[1026]; size_t send_len = std::min(len, static_cast<size_t>(1024));
diff --git a/fpdfsdk/cpdfsdk_interform.cpp b/fpdfsdk/cpdfsdk_interform.cpp index 3cfc84d..6a4b591 100644 --- a/fpdfsdk/cpdfsdk_interform.cpp +++ b/fpdfsdk/cpdfsdk_interform.cpp
@@ -506,10 +506,10 @@ continue; WideString name; name = pField->GetUnicodeTextFor("T"); - ByteString name_b = ByteString::FromUnicode(name); + ByteString name_b = name.ToDefANSI(); ByteString csBValue = pField->GetStringFor("V"); WideString csWValue = PDF_DecodeText(csBValue); - ByteString csValue_b = ByteString::FromUnicode(csWValue); + ByteString csValue_b = csWValue.ToDefANSI(); fdfEncodedData << name_b << "=" << csValue_b; if (i != pFields->GetCount() - 1) fdfEncodedData << "&";
diff --git a/fxbarcode/BC_UtilCodingConvert.cpp b/fxbarcode/BC_UtilCodingConvert.cpp index 77c52f2..96aeadd 100644 --- a/fxbarcode/BC_UtilCodingConvert.cpp +++ b/fxbarcode/BC_UtilCodingConvert.cpp
@@ -12,7 +12,7 @@ void CBC_UtilCodingConvert::UnicodeToLocale(const WideString& src, ByteString& dst) { - dst = ByteString::FromUnicode(src); + dst = src.ToDefANSI(); } void CBC_UtilCodingConvert::LocaleToUtf8(const ByteString& src, @@ -34,15 +34,13 @@ for (uint8_t value : src) utf8 += value; - WideString unicode = WideString::FromUTF8(utf8.AsStringView()); - dst = ByteString::FromUnicode(unicode); + dst = WideString::FromUTF8(utf8.AsStringView()).ToDefANSI(); } void CBC_UtilCodingConvert::Utf8ToLocale(const uint8_t* src, int32_t count, ByteString& dst) { - WideString unicode = WideString::FromUTF8(ByteStringView(src, count)); - dst = ByteString::FromUnicode(unicode); + dst = WideString::FromUTF8(ByteStringView(src, count)).ToDefANSI(); } void CBC_UtilCodingConvert::UnicodeToUTF8(const WideString& src,
diff --git a/fxjs/cfxjse_class.cpp b/fxjs/cfxjse_class.cpp index e7e54c8..9207fd5 100644 --- a/fxjs/cfxjse_class.cpp +++ b/fxjs/cfxjse_class.cpp
@@ -112,8 +112,7 @@ WideString err = JSFormatErrorString(pClassDescriptor->name, *szPropName, result.Error()); v8::MaybeLocal<v8::String> str = v8::String::NewFromUtf8( - info.GetIsolate(), ByteString::FromUnicode(err).c_str(), - v8::NewStringType::kNormal); + info.GetIsolate(), err.ToDefANSI().c_str(), v8::NewStringType::kNormal); info.GetIsolate()->ThrowException(str.ToLocalChecked()); return; }
diff --git a/fxjs/cjs_field.cpp b/fxjs/cjs_field.cpp index 19e6712..cd97372 100644 --- a/fxjs/cjs_field.cpp +++ b/fxjs/cjs_field.cpp
@@ -731,7 +731,7 @@ if (!m_bCanSet) return CJS_Return(JSMessage::kReadOnlyError); - ByteString byte_str = ByteString::FromUnicode(pRuntime->ToWideString(vp)); + ByteString byte_str = pRuntime->ToWideString(vp).ToDefANSI(); if (m_bDelay) { AddDelay_String(FP_BORDERSTYLE, byte_str); } else { @@ -2002,7 +2002,7 @@ if (!m_bCanSet) return CJS_Return(JSMessage::kReadOnlyError); - if (ByteString::FromUnicode(pRuntime->ToWideString(vp)).IsEmpty()) + if (pRuntime->ToWideString(vp).ToDefANSI().IsEmpty()) return CJS_Return(JSMessage::kValueError); return CJS_Return(); }
diff --git a/fxjs/cjs_global.cpp b/fxjs/cjs_global.cpp index 567d548..1865f57 100644 --- a/fxjs/cjs_global.cpp +++ b/fxjs/cjs_global.cpp
@@ -232,7 +232,7 @@ CJS_Return CJS_Global::DelProperty(CJS_Runtime* pRuntime, const wchar_t* propname) { - auto it = m_MapGlobal.find(ByteString::FromUnicode(propname)); + auto it = m_MapGlobal.find(WideString(propname).ToDefANSI()); if (it == m_MapGlobal.end()) return CJS_Return(JSMessage::kUnknownProperty); @@ -242,7 +242,7 @@ CJS_Return CJS_Global::GetProperty(CJS_Runtime* pRuntime, const wchar_t* propname) { - auto it = m_MapGlobal.find(ByteString::FromUnicode(propname)); + auto it = m_MapGlobal.find(WideString(propname).ToDefANSI()); if (it == m_MapGlobal.end()) return CJS_Return(); @@ -272,7 +272,7 @@ CJS_Return CJS_Global::SetProperty(CJS_Runtime* pRuntime, const wchar_t* propname, v8::Local<v8::Value> vp) { - ByteString sPropName = ByteString::FromUnicode(propname); + ByteString sPropName = WideString(propname).ToDefANSI(); if (vp->IsNumber()) { return SetGlobalVariables(sPropName, JS_GlobalDataType::NUMBER, pRuntime->ToDouble(vp), false, "", @@ -284,10 +284,9 @@ v8::Local<v8::Object>(), false); } if (vp->IsString()) { - return SetGlobalVariables( - sPropName, JS_GlobalDataType::STRING, 0, false, - ByteString::FromUnicode(pRuntime->ToWideString(vp)), - v8::Local<v8::Object>(), false); + return SetGlobalVariables(sPropName, JS_GlobalDataType::STRING, 0, false, + pRuntime->ToWideString(vp).ToDefANSI(), + v8::Local<v8::Object>(), false); } if (vp->IsObject()) { return SetGlobalVariables(sPropName, JS_GlobalDataType::OBJECT, 0, false, @@ -310,8 +309,7 @@ if (params.size() != 2) return CJS_Return(JSMessage::kParamError); - auto it = m_MapGlobal.find( - ByteString::FromUnicode(pRuntime->ToWideString(params[0]))); + auto it = m_MapGlobal.find(pRuntime->ToWideString(params[0]).ToDefANSI()); if (it == m_MapGlobal.end() || it->second->bDeleted) return CJS_Return(JSMessage::kGlobalNotFoundError); @@ -432,7 +430,7 @@ continue; } if (v->IsString()) { - ByteString sValue = ByteString::FromUnicode(pRuntime->ToWideString(v)); + ByteString sValue = pRuntime->ToWideString(v).ToDefANSI(); CJS_KeyValue* pObjElement = new CJS_KeyValue; pObjElement->nType = JS_GlobalDataType::STRING; pObjElement->sKey = sKey;
diff --git a/fxjs/cjs_publicmethods.cpp b/fxjs/cjs_publicmethods.cpp index a86ece3..f2cd3ff 100644 --- a/fxjs/cjs_publicmethods.cpp +++ b/fxjs/cjs_publicmethods.cpp
@@ -327,7 +327,7 @@ return pRuntime->ToArray(val); WideString wsStr = pRuntime->ToWideString(val); - ByteString t = ByteString::FromUnicode(wsStr); + ByteString t = wsStr.ToDefANSI(); const char* p = t.c_str(); int nIndex = 0; @@ -882,7 +882,7 @@ return CJS_Return(L"No event handler"); WideString& Value = pEvent->Value(); - ByteString strValue = StrTrim(ByteString::FromUnicode(Value)); + ByteString strValue = StrTrim(Value.ToDefANSI()); if (strValue.IsEmpty()) return CJS_Return(); @@ -1078,7 +1078,7 @@ return CJS_Return(JSMessage::kBadObjectError); WideString& Value = pEvent->Value(); - ByteString strValue = StrTrim(ByteString::FromUnicode(Value)); + ByteString strValue = StrTrim(Value.ToDefANSI()); if (strValue.IsEmpty()) return CJS_Return(); @@ -1683,7 +1683,7 @@ if (pEvent->Value().IsEmpty()) return CJS_Return(); - double dEentValue = atof(ByteString::FromUnicode(pEvent->Value()).c_str()); + double dEentValue = atof(pEvent->Value().ToDefANSI().c_str()); bool bGreaterThan = pRuntime->ToBoolean(params[0]); double dGreaterThan = pRuntime->ToDouble(params[1]); bool bLessThan = pRuntime->ToBoolean(params[2]);
diff --git a/fxjs/cjs_runtime.cpp b/fxjs/cjs_runtime.cpp index 9329a48..6042538 100644 --- a/fxjs/cjs_runtime.cpp +++ b/fxjs/cjs_runtime.cpp
@@ -234,7 +234,7 @@ v8::Local<v8::Value> value) { bool bAllowNaN = false; if (value->IsString()) { - ByteString bstr = ByteString::FromUnicode(ToWideString(value)); + ByteString bstr = ToWideString(value).ToDefANSI(); if (bstr.GetLength() == 0) return value; if (bstr == "NaN")
diff --git a/xfa/fgas/font/cfgas_fontmgr.cpp b/xfa/fgas/font/cfgas_fontmgr.cpp index 5ab91b3..f173bf2 100644 --- a/xfa/fgas/font/cfgas_fontmgr.cpp +++ b/xfa/fgas/font/cfgas_fontmgr.cpp
@@ -416,8 +416,7 @@ } ByteString bsName; bool bFolder; - ByteString bsFolderSeparator = - ByteString::FromUnicode(WideString(kFolderSeparator)); + ByteString bsFolderSeparator = WideString(kFolderSeparator).ToDefANSI(); while (true) { if (!FX_GetNextFile(pCurHandle, &bsName, &bFolder)) { FX_CloseFolder(pCurHandle);
diff --git a/xfa/fgas/font/cfgas_gefont.cpp b/xfa/fgas/font/cfgas_gefont.cpp index 740504f..b5cb796 100644 --- a/xfa/fgas/font/cfgas_gefont.cpp +++ b/xfa/fgas/font/cfgas_gefont.cpp
@@ -76,7 +76,7 @@ return false; ByteString csFontFamily; if (pszFontFamily) - csFontFamily = ByteString::FromUnicode(pszFontFamily); + csFontFamily = WideString(pszFontFamily).ToDefANSI(); int32_t iWeight = FontStyleIsBold(dwFontStyles) ? FXFONT_FW_BOLD : FXFONT_FW_NORMAL;
diff --git a/xfa/fgas/font/cfgas_pdffontmgr.cpp b/xfa/fgas/font/cfgas_pdffontmgr.cpp index 80d6196..4220d4d 100644 --- a/xfa/fgas/font/cfgas_pdffontmgr.cpp +++ b/xfa/fgas/font/cfgas_pdffontmgr.cpp
@@ -80,7 +80,7 @@ if (it != m_FontMap.end()) return it->second; - ByteString bsPsName = ByteString::FromUnicode(WideString(wsFontFamily)); + ByteString bsPsName = WideString(wsFontFamily).ToDefANSI(); bool bBold = FontStyleIsBold(dwFontStyles); bool bItalic = FontStyleIsItalic(dwFontStyles); ByteString strFontName = PsNameToFontName(bsPsName, bBold, bItalic);
diff --git a/xfa/fxfa/parser/cxfa_node.cpp b/xfa/fxfa/parser/cxfa_node.cpp index 7ff205c..8babe45 100644 --- a/xfa/fxfa/parser/cxfa_node.cpp +++ b/xfa/fxfa/parser/cxfa_node.cpp
@@ -214,7 +214,7 @@ pdfium::MakeRetain<CFX_MemoryStream>(pImageBuffer, iRead, false); } } else { - bsContent = ByteString::FromUnicode(wsImage); + bsContent = wsImage.ToDefANSI(); pImageFileRead = pdfium::MakeRetain<CFX_MemoryStream>( const_cast<uint8_t*>(bsContent.raw_str()), bsContent.GetLength(), false);