| // Copyright 2013 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "components/dom_distiller/core/distiller.h" |
| |
| #include <map> |
| #include <vector> |
| |
| #include "base/auto_reset.h" |
| #include "base/bind.h" |
| #include "base/callback.h" |
| #include "base/location.h" |
| #include "base/message_loop/message_loop.h" |
| #include "base/strings/string_number_conversions.h" |
| #include "base/strings/utf_string_conversions.h" |
| #include "base/values.h" |
| #include "components/dom_distiller/core/distiller_page.h" |
| #include "components/dom_distiller/core/distiller_url_fetcher.h" |
| #include "components/dom_distiller/core/proto/distilled_article.pb.h" |
| #include "components/dom_distiller/core/proto/distilled_page.pb.h" |
| #include "net/url_request/url_request_context_getter.h" |
| |
| namespace { |
| // Maximum number of distilled pages in an article. |
| const size_t kMaxPagesInArticle = 32; |
| } |
| |
| namespace dom_distiller { |
| |
| DistillerFactoryImpl::DistillerFactoryImpl( |
| scoped_ptr<DistillerURLFetcherFactory> distiller_url_fetcher_factory, |
| const dom_distiller::proto::DomDistillerOptions& dom_distiller_options) |
| : distiller_url_fetcher_factory_(distiller_url_fetcher_factory.Pass()), |
| dom_distiller_options_(dom_distiller_options) { |
| } |
| |
| DistillerFactoryImpl::~DistillerFactoryImpl() {} |
| |
| scoped_ptr<Distiller> DistillerFactoryImpl::CreateDistiller() { |
| scoped_ptr<DistillerImpl> distiller(new DistillerImpl( |
| *distiller_url_fetcher_factory_, dom_distiller_options_)); |
| return distiller.PassAs<Distiller>(); |
| } |
| |
| DistillerImpl::DistilledPageData::DistilledPageData() {} |
| |
| DistillerImpl::DistilledPageData::~DistilledPageData() {} |
| |
| DistillerImpl::DistillerImpl( |
| const DistillerURLFetcherFactory& distiller_url_fetcher_factory, |
| const dom_distiller::proto::DomDistillerOptions& dom_distiller_options) |
| : distiller_url_fetcher_factory_(distiller_url_fetcher_factory), |
| dom_distiller_options_(dom_distiller_options), |
| max_pages_in_article_(kMaxPagesInArticle), |
| destruction_allowed_(true), |
| weak_factory_(this) { |
| } |
| |
| DistillerImpl::~DistillerImpl() { |
| DCHECK(destruction_allowed_); |
| } |
| |
| void DistillerImpl::SetMaxNumPagesInArticle(size_t max_num_pages) { |
| max_pages_in_article_ = max_num_pages; |
| } |
| |
| bool DistillerImpl::AreAllPagesFinished() const { |
| return started_pages_index_.empty() && waiting_pages_.empty(); |
| } |
| |
| size_t DistillerImpl::TotalPageCount() const { |
| return waiting_pages_.size() + started_pages_index_.size() + |
| finished_pages_index_.size(); |
| } |
| |
| void DistillerImpl::AddToDistillationQueue(int page_num, const GURL& url) { |
| if (!IsPageNumberInUse(page_num) && url.is_valid() && |
| TotalPageCount() < max_pages_in_article_ && |
| seen_urls_.find(url.spec()) == seen_urls_.end()) { |
| waiting_pages_[page_num] = url; |
| } |
| } |
| |
| bool DistillerImpl::IsPageNumberInUse(int page_num) const { |
| return waiting_pages_.find(page_num) != waiting_pages_.end() || |
| started_pages_index_.find(page_num) != started_pages_index_.end() || |
| finished_pages_index_.find(page_num) != finished_pages_index_.end(); |
| } |
| |
| DistillerImpl::DistilledPageData* DistillerImpl::GetPageAtIndex(size_t index) |
| const { |
| DCHECK_LT(index, pages_.size()); |
| DistilledPageData* page_data = pages_[index]; |
| DCHECK(page_data); |
| return page_data; |
| } |
| |
| void DistillerImpl::DistillPage(const GURL& url, |
| scoped_ptr<DistillerPage> distiller_page, |
| const DistillationFinishedCallback& finished_cb, |
| const DistillationUpdateCallback& update_cb) { |
| DCHECK(AreAllPagesFinished()); |
| distiller_page_ = distiller_page.Pass(); |
| finished_cb_ = finished_cb; |
| update_cb_ = update_cb; |
| |
| AddToDistillationQueue(0, url); |
| DistillNextPage(); |
| } |
| |
| void DistillerImpl::DistillNextPage() { |
| if (!waiting_pages_.empty()) { |
| std::map<int, GURL>::iterator front = waiting_pages_.begin(); |
| int page_num = front->first; |
| const GURL url = front->second; |
| |
| waiting_pages_.erase(front); |
| DCHECK(url.is_valid()); |
| DCHECK(started_pages_index_.find(page_num) == started_pages_index_.end()); |
| DCHECK(finished_pages_index_.find(page_num) == finished_pages_index_.end()); |
| seen_urls_.insert(url.spec()); |
| pages_.push_back(new DistilledPageData()); |
| started_pages_index_[page_num] = pages_.size() - 1; |
| distiller_page_->DistillPage( |
| url, |
| dom_distiller_options_, |
| base::Bind(&DistillerImpl::OnPageDistillationFinished, |
| weak_factory_.GetWeakPtr(), |
| page_num, |
| url)); |
| } |
| } |
| |
| void DistillerImpl::OnPageDistillationFinished( |
| int page_num, |
| const GURL& page_url, |
| scoped_ptr<DistilledPageInfo> distilled_page, |
| bool distillation_successful) { |
| DCHECK(distilled_page.get()); |
| DCHECK(started_pages_index_.find(page_num) != started_pages_index_.end()); |
| if (distillation_successful) { |
| DistilledPageData* page_data = |
| GetPageAtIndex(started_pages_index_[page_num]); |
| page_data->distilled_page_proto = |
| new base::RefCountedData<DistilledPageProto>(); |
| page_data->page_num = page_num; |
| page_data->distilled_page_proto->data.set_title(distilled_page->title); |
| page_data->distilled_page_proto->data.set_url(page_url.spec()); |
| page_data->distilled_page_proto->data.set_html(distilled_page->html); |
| |
| GURL next_page_url(distilled_page->next_page_url); |
| if (next_page_url.is_valid()) { |
| // The pages should be in same origin. |
| DCHECK_EQ(next_page_url.GetOrigin(), page_url.GetOrigin()); |
| AddToDistillationQueue(page_num + 1, next_page_url); |
| } |
| |
| GURL prev_page_url(distilled_page->prev_page_url); |
| if (prev_page_url.is_valid()) { |
| DCHECK_EQ(prev_page_url.GetOrigin(), page_url.GetOrigin()); |
| AddToDistillationQueue(page_num - 1, prev_page_url); |
| } |
| |
| for (size_t img_num = 0; img_num < distilled_page->image_urls.size(); |
| ++img_num) { |
| std::string image_id = |
| base::IntToString(page_num + 1) + "_" + base::IntToString(img_num); |
| FetchImage(page_num, image_id, distilled_page->image_urls[img_num]); |
| } |
| |
| AddPageIfDone(page_num); |
| DistillNextPage(); |
| } else { |
| started_pages_index_.erase(page_num); |
| RunDistillerCallbackIfDone(); |
| } |
| } |
| |
| void DistillerImpl::FetchImage(int page_num, |
| const std::string& image_id, |
| const std::string& item) { |
| DCHECK(started_pages_index_.find(page_num) != started_pages_index_.end()); |
| DistilledPageData* page_data = GetPageAtIndex(started_pages_index_[page_num]); |
| DistillerURLFetcher* fetcher = |
| distiller_url_fetcher_factory_.CreateDistillerURLFetcher(); |
| page_data->image_fetchers_.push_back(fetcher); |
| |
| fetcher->FetchURL(item, |
| base::Bind(&DistillerImpl::OnFetchImageDone, |
| weak_factory_.GetWeakPtr(), |
| page_num, |
| base::Unretained(fetcher), |
| image_id)); |
| } |
| |
| void DistillerImpl::OnFetchImageDone(int page_num, |
| DistillerURLFetcher* url_fetcher, |
| const std::string& id, |
| const std::string& response) { |
| DCHECK(started_pages_index_.find(page_num) != started_pages_index_.end()); |
| DistilledPageData* page_data = GetPageAtIndex(started_pages_index_[page_num]); |
| DCHECK(page_data->distilled_page_proto); |
| DCHECK(url_fetcher); |
| ScopedVector<DistillerURLFetcher>::iterator fetcher_it = |
| std::find(page_data->image_fetchers_.begin(), |
| page_data->image_fetchers_.end(), |
| url_fetcher); |
| |
| DCHECK(fetcher_it != page_data->image_fetchers_.end()); |
| // Delete the |url_fetcher| by DeleteSoon since the OnFetchImageDone |
| // callback is invoked by the |url_fetcher|. |
| page_data->image_fetchers_.weak_erase(fetcher_it); |
| base::MessageLoop::current()->DeleteSoon(FROM_HERE, url_fetcher); |
| |
| DistilledPageProto_Image* image = |
| page_data->distilled_page_proto->data.add_image(); |
| image->set_name(id); |
| image->set_data(response); |
| |
| AddPageIfDone(page_num); |
| } |
| |
| void DistillerImpl::AddPageIfDone(int page_num) { |
| DCHECK(started_pages_index_.find(page_num) != started_pages_index_.end()); |
| DCHECK(finished_pages_index_.find(page_num) == finished_pages_index_.end()); |
| DistilledPageData* page_data = GetPageAtIndex(started_pages_index_[page_num]); |
| if (page_data->image_fetchers_.empty()) { |
| finished_pages_index_[page_num] = started_pages_index_[page_num]; |
| started_pages_index_.erase(page_num); |
| const ArticleDistillationUpdate& article_update = |
| CreateDistillationUpdate(); |
| DCHECK_EQ(article_update.GetPagesSize(), finished_pages_index_.size()); |
| update_cb_.Run(article_update); |
| RunDistillerCallbackIfDone(); |
| } |
| } |
| |
| const ArticleDistillationUpdate DistillerImpl::CreateDistillationUpdate() |
| const { |
| bool has_prev_page = false; |
| bool has_next_page = false; |
| if (!finished_pages_index_.empty()) { |
| int prev_page_num = finished_pages_index_.begin()->first - 1; |
| int next_page_num = finished_pages_index_.rbegin()->first + 1; |
| has_prev_page = IsPageNumberInUse(prev_page_num); |
| has_next_page = IsPageNumberInUse(next_page_num); |
| } |
| |
| std::vector<scoped_refptr<ArticleDistillationUpdate::RefCountedPageProto> > |
| update_pages; |
| for (std::map<int, size_t>::const_iterator it = finished_pages_index_.begin(); |
| it != finished_pages_index_.end(); |
| ++it) { |
| update_pages.push_back(pages_[it->second]->distilled_page_proto); |
| } |
| return ArticleDistillationUpdate(update_pages, has_next_page, has_prev_page); |
| } |
| |
| void DistillerImpl::RunDistillerCallbackIfDone() { |
| DCHECK(!finished_cb_.is_null()); |
| if (AreAllPagesFinished()) { |
| bool first_page = true; |
| scoped_ptr<DistilledArticleProto> article_proto( |
| new DistilledArticleProto()); |
| // Stitch the pages back into the article. |
| for (std::map<int, size_t>::iterator it = finished_pages_index_.begin(); |
| it != finished_pages_index_.end();) { |
| DistilledPageData* page_data = GetPageAtIndex(it->second); |
| *(article_proto->add_pages()) = page_data->distilled_page_proto->data; |
| |
| if (first_page) { |
| article_proto->set_title(page_data->distilled_page_proto->data.title()); |
| first_page = false; |
| } |
| |
| finished_pages_index_.erase(it++); |
| } |
| |
| pages_.clear(); |
| DCHECK_LE(static_cast<size_t>(article_proto->pages_size()), |
| max_pages_in_article_); |
| |
| DCHECK(pages_.empty()); |
| DCHECK(finished_pages_index_.empty()); |
| |
| base::AutoReset<bool> dont_delete_this_in_callback(&destruction_allowed_, |
| false); |
| finished_cb_.Run(article_proto.Pass()); |
| finished_cb_.Reset(); |
| } |
| } |
| |
| } // namespace dom_distiller |