// Copyright 2017 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "services/shape_detection/text_detection_impl_win.h"
#include <windows.foundation.collections.h>
#include <windows.globalization.h>
#include <memory>
#include <string>
#include "base/functional/bind.h"
#include "base/logging.h"
#include "base/win/core_winrt_util.h"
#include "base/win/post_async_results.h"
#include "base/win/scoped_hstring.h"
#include "mojo/public/cpp/bindings/pending_receiver.h"
#include "mojo/public/cpp/bindings/self_owned_receiver.h"
#include "services/shape_detection/detection_utils_win.h"
#include "services/shape_detection/text_detection_impl.h"
#include "ui/gfx/geometry/rect_f.h"
namespace shape_detection {
using ABI::Windows::Foundation::IAsyncOperation;
using ABI::Windows::Foundation::Collections::IVectorView;
using ABI::Windows::Globalization::ILanguageFactory;
using ABI::Windows::Graphics::Imaging::ISoftwareBitmap;
using ABI::Windows::Graphics::Imaging::ISoftwareBitmapStatics;
using ABI::Windows::Media::Ocr::IOcrEngine;
using ABI::Windows::Media::Ocr::IOcrEngineStatics;
using ABI::Windows::Media::Ocr::IOcrLine;
using ABI::Windows::Media::Ocr::IOcrResult;
using ABI::Windows::Media::Ocr::IOcrWord;
using ABI::Windows::Media::Ocr::OcrLine;
using ABI::Windows::Media::Ocr::OcrResult;
using ABI::Windows::Media::Ocr::OcrWord;
using base::win::GetActivationFactory;
using base::win::ScopedHString;
using Microsoft::WRL::ComPtr;
// static
void TextDetectionImpl::Create(
mojo::PendingReceiver<mojom::TextDetection> receiver) {
// Text Detection specification only supports Latin-1 text as documented in
// https://wicg.github.io/shape-detection-api/text.html#text-detection-api.
// TODO(junwei.fu): https://crbug.com/794097 consider supporting other Latin
// script language.
ScopedHString language_hstring = ScopedHString::Create("en");
if (!language_hstring.is_valid())
return;
ComPtr<ILanguageFactory> language_factory;
HRESULT hr =
GetActivationFactory<ILanguageFactory,
RuntimeClass_Windows_Globalization_Language>(
&language_factory);
if (FAILED(hr)) {
DLOG(ERROR) << "ILanguage factory failed: "
<< logging::SystemErrorCodeToString(hr);
return;
}
ComPtr<ABI::Windows::Globalization::ILanguage> language;
hr = language_factory->CreateLanguage(language_hstring.get(), &language);
if (FAILED(hr)) {
DLOG(ERROR) << "Create language failed: "
<< logging::SystemErrorCodeToString(hr);
return;
}
ComPtr<IOcrEngineStatics> engine_factory;
hr = GetActivationFactory<IOcrEngineStatics,
RuntimeClass_Windows_Media_Ocr_OcrEngine>(
&engine_factory);
if (FAILED(hr)) {
DLOG(ERROR) << "IOcrEngineStatics factory failed: "
<< logging::SystemErrorCodeToString(hr);
return;
}
boolean is_supported = false;
hr = engine_factory->IsLanguageSupported(language.Get(), &is_supported);
if (FAILED(hr) || !is_supported)
return;
ComPtr<IOcrEngine> ocr_engine;
hr = engine_factory->TryCreateFromLanguage(language.Get(), &ocr_engine);
if (FAILED(hr)) {
DLOG(ERROR) << "Create engine failed from language: "
<< logging::SystemErrorCodeToString(hr);
return;
}
ComPtr<ISoftwareBitmapStatics> bitmap_factory;
hr = GetActivationFactory<
ISoftwareBitmapStatics,
RuntimeClass_Windows_Graphics_Imaging_SoftwareBitmap>(&bitmap_factory);
if (FAILED(hr)) {
DLOG(ERROR) << "ISoftwareBitmapStatics factory failed: "
<< logging::SystemErrorCodeToString(hr);
return;
}
auto impl = std::make_unique<TextDetectionImplWin>(std::move(ocr_engine),
std::move(bitmap_factory));
auto* impl_ptr = impl.get();
impl_ptr->SetReceiver(
mojo::MakeSelfOwnedReceiver(std::move(impl), std::move(receiver)));
}
TextDetectionImplWin::TextDetectionImplWin(
ComPtr<IOcrEngine> ocr_engine,
ComPtr<ISoftwareBitmapStatics> bitmap_factory)
: ocr_engine_(std::move(ocr_engine)),
bitmap_factory_(std::move(bitmap_factory)) {
DCHECK(ocr_engine_);
DCHECK(bitmap_factory_);
}
TextDetectionImplWin::~TextDetectionImplWin() = default;
void TextDetectionImplWin::Detect(const SkBitmap& bitmap,
DetectCallback callback) {
if (FAILED(BeginDetect(bitmap))) {
// No detection taking place; run |callback| with an empty array of results.
std::move(callback).Run(std::vector<mojom::TextDetectionResultPtr>());
return;
}
// Hold on the callback until AsyncOperation completes.
recognize_text_callback_ = std::move(callback);
// This prevents the Detect function from being called before the
// AsyncOperation completes.
receiver_->PauseIncomingMethodCallProcessing();
}
HRESULT TextDetectionImplWin::BeginDetect(const SkBitmap& bitmap) {
ComPtr<ISoftwareBitmap> win_bitmap =
CreateWinBitmapFromSkBitmap(bitmap, bitmap_factory_.Get());
if (!win_bitmap)
return E_FAIL;
// Recognize text asynchronously.
ComPtr<IAsyncOperation<OcrResult*>> async_op;
const HRESULT hr = ocr_engine_->RecognizeAsync(win_bitmap.Get(), &async_op);
if (FAILED(hr)) {
DLOG(ERROR) << "Recognize text asynchronously failed: "
<< logging::SystemErrorCodeToString(hr);
return hr;
}
// Use WeakPtr to bind the callback so that the once callback will not be run
// if this object has been already destroyed. |win_bitmap| needs to be kept
// alive until OnTextDetected().
return base::win::PostAsyncResults(
std::move(async_op),
base::BindOnce(&TextDetectionImplWin::OnTextDetected,
weak_factory_.GetWeakPtr(), std::move(win_bitmap)));
}
std::vector<mojom::TextDetectionResultPtr>
TextDetectionImplWin::BuildTextDetectionResult(ComPtr<IOcrResult> ocr_result) {
std::vector<mojom::TextDetectionResultPtr> results;
if (!ocr_result)
return results;
ComPtr<IVectorView<OcrLine*>> ocr_lines;
HRESULT hr = ocr_result->get_Lines(&ocr_lines);
if (FAILED(hr)) {
DLOG(ERROR) << "Get Lines failed: " << logging::SystemErrorCodeToString(hr);
return results;
}
uint32_t count;
hr = ocr_lines->get_Size(&count);
if (FAILED(hr)) {
DLOG(ERROR) << "get_Size failed: " << logging::SystemErrorCodeToString(hr);
return results;
}
results.reserve(count);
for (uint32_t i = 0; i < count; ++i) {
ComPtr<IOcrLine> line;
hr = ocr_lines->GetAt(i, &line);
if (FAILED(hr))
break;
HSTRING text;
hr = line->get_Text(&text);
if (FAILED(hr))
break;
// Gets bounding box with the words detected in the current line of Text.
ComPtr<IVectorView<OcrWord*>> ocr_words;
hr = line->get_Words(&ocr_words);
if (FAILED(hr))
break;
uint32_t words_count;
hr = ocr_words->get_Size(&words_count);
if (FAILED(hr))
break;
auto result = shape_detection::mojom::TextDetectionResult::New();
for (uint32_t word_num = 0; word_num < words_count; ++word_num) {
ComPtr<IOcrWord> word;
hr = ocr_words->GetAt(word_num, &word);
if (FAILED(hr))
break;
ABI::Windows::Foundation::Rect bounds;
hr = word->get_BoundingRect(&bounds);
if (FAILED(hr))
break;
result->bounding_box = gfx::UnionRects(
result->bounding_box,
gfx::RectF(bounds.X, bounds.Y, bounds.Width, bounds.Height));
}
result->raw_value = ScopedHString(text).GetAsUTF8();
results.push_back(std::move(result));
}
return results;
}
// |win_bitmap| is passed here so that it is kept alive until the AsyncOperation
// completes because RecognizeAsync does not hold a reference.
void TextDetectionImplWin::OnTextDetected(
ComPtr<ISoftwareBitmap> /* win_bitmap */,
ComPtr<IOcrResult> ocr_result) {
std::move(recognize_text_callback_)
.Run(BuildTextDetectionResult(std::move(ocr_result)));
receiver_->ResumeIncomingMethodCallProcessing();
}
} // namespace shape_detection
Codebase Browser
chromium