Skip to content
This repository has been archived by the owner on Mar 9, 2021. It is now read-only.

Commit

Permalink
Merge branch 'master' into release/api
Browse files Browse the repository at this point in the history
  • Loading branch information
@johanneszab
johanneszab committed Dec 11, 2018
2 parents 6177237 + 163bbb1 commit 2622a52
Show file tree
Hide file tree
Showing 9 changed files with 32 additions and 24 deletions.
4 changes: 2 additions & 2 deletions src/TumblThree/SharedAssemblyInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@

[assembly: ComVisible(false)]
[assembly: NeutralResourcesLanguage("en-US", UltimateResourceFallbackLocation.MainAssembly)]
[assembly: AssemblyVersion("1.0.8.67")]
[assembly: AssemblyFileVersion("1.0.8.67")]
[assembly: AssemblyVersion("1.0.8.68")]
[assembly: AssemblyFileVersion("1.0.8.68")]
5 changes: 4 additions & 1 deletion src/TumblThree/TumblThree.Applications/Crawler/AbstractCrawler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,9 @@ public virtual async Task IsBlogOnlineAsync()
}
catch (WebException webException)
{
if (webException.Status == WebExceptionStatus.RequestCanceled)
return;

Logger.Error("AbstractCrawler:IsBlogOnlineAsync:WebException {0}", webException);
shellService.ShowError(webException, Resources.BlogIsOffline, blog.Name);
blog.Online = false;
Expand Down Expand Up @@ -222,7 +225,7 @@ protected int DetermineDuplicates<T>() => statisticsBag.Where(url => url.GetType

protected void CleanCollectedBlogStatistics() => statisticsBag = null;

protected bool CheckifShouldStop() => ct.IsCancellationRequested;
protected bool CheckIfShouldStop() => ct.IsCancellationRequested;

protected void CheckIfShouldPause()
{
Expand Down
2 changes: 1 addition & 1 deletion src/TumblThree/TumblThree.Applications/Crawler/AbstractTumblrCrawler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ protected async Task<string> UpdateTumblrKeyAsync(string url)
string document = await GetRequestAsync(url);
return ExtractTumblrKey(document);
}
catch (WebException webException) when (webException.Response == null && webException.Status == WebExceptionStatus.RequestCanceled)
catch (WebException webException) when (webException.Status == WebExceptionStatus.RequestCanceled)
{
return string.Empty;
}
Expand Down
16 changes: 9 additions & 7 deletions src/TumblThree/TumblThree.Applications/Crawler/TumblrBlogCrawler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
using System.IO;
using System.Linq;
using System.Net;
using System.Runtime.Remoting.Messaging;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
Expand Down Expand Up @@ -66,7 +65,7 @@ public override async Task IsBlogOnlineAsync()
}
catch (WebException webException)
{
if (webException.Response == null && webException.Status == WebExceptionStatus.RequestCanceled)
if (webException.Status == WebExceptionStatus.RequestCanceled)
return;

if (HandleUnauthorizedWebException(webException))
Expand All @@ -93,8 +92,11 @@ public override async Task UpdateMetaInformationAsync()
{
await UpdateMetaInformationCoreAsync();
}
catch (WebException webException) when ((webException.Response != null))
catch (WebException webException)
{
if (webException.Status == WebExceptionStatus.RequestCanceled)
return;

HandleLimitExceededWebException(webException);
}
}
Expand Down Expand Up @@ -220,7 +222,7 @@ private async Task UpdateTotalPostCountAsync()
}
catch (WebException webException)
{
if (webException.Response == null && webException.Status == WebExceptionStatus.RequestCanceled)
if (webException.Status == WebExceptionStatus.RequestCanceled)
return;

HandleLimitExceededWebException(webException);
Expand Down Expand Up @@ -249,7 +251,7 @@ private async Task<ulong> GetHighestPostIdAsync()
}
catch (WebException webException)
{
if (webException.Response == null && webException.Status == WebExceptionStatus.RequestCanceled)
if (webException.Status == WebExceptionStatus.RequestCanceled)
return 0;

HandleLimitExceededWebException(webException);
Expand Down Expand Up @@ -305,7 +307,7 @@ private async Task<bool> GetUrlsAsync()
break;
}

if (CheckifShouldStop())
if (CheckIfShouldStop())
break;

CheckIfShouldPause();
Expand Down Expand Up @@ -337,7 +339,7 @@ private async Task CrawlPageAsync(int pageNumber)
numberOfPagesCrawled += blog.PageSize;
UpdateProgressQueueInformation(Resources.ProgressGetUrlLong, numberOfPagesCrawled, blog.Posts);
}
catch (WebException webException) when ((webException.Response != null))
catch (WebException webException) when (webException.Response != null)
{
if (HandleLimitExceededWebException(webException))
incompleteCrawl = true;
Expand Down
12 changes: 6 additions & 6 deletions src/TumblThree/TumblThree.Applications/Crawler/TumblrHiddenCrawler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ public override async Task IsBlogOnlineAsync()
}
catch (WebException webException)
{
if (webException.Response == null && webException.Status == WebExceptionStatus.RequestCanceled)
if (webException.Status == WebExceptionStatus.RequestCanceled)
return;

if (HandleServiceUnavailableWebException(webException))
Expand Down Expand Up @@ -104,7 +104,7 @@ public override async Task UpdateMetaInformationAsync()
}
catch (WebException webException)
{
if (webException.Response == null && webException.Status == WebExceptionStatus.RequestCanceled)
if (webException.Status == WebExceptionStatus.RequestCanceled)
return;

HandleServiceUnavailableWebException(webException);
Expand Down Expand Up @@ -200,7 +200,7 @@ private async Task CrawlPageAsync(int pageNumber)
var response = ConvertJsonToClass<TumblrJson>(document);
await AddUrlsToDownloadListAsync(response, pageNumber);
}
catch (WebException webException) when ((webException.Response != null))
catch (WebException webException) when (webException.Response != null)
{
if (HandleLimitExceededWebException(webException))
incompleteCrawl = true;
Expand All @@ -227,7 +227,7 @@ private async Task<ulong> GetHighestPostIdAsync()
}
catch (WebException webException)
{
if (webException.Response == null && webException.Status == WebExceptionStatus.RequestCanceled)
if (webException.Status == WebExceptionStatus.RequestCanceled)
return 0;

HandleLimitExceededWebException(webException);
Expand Down Expand Up @@ -284,7 +284,7 @@ private async Task<bool> CheckIfLoggedInAsync()
}
catch (WebException webException)
{
if (webException.Response == null && webException.Status == WebExceptionStatus.RequestCanceled)
if (webException.Status == WebExceptionStatus.RequestCanceled)
return true;

if (HandleServiceUnavailableWebException(webException))
Expand Down Expand Up @@ -333,7 +333,7 @@ private async Task AddUrlsToDownloadListAsync(TumblrJson response, int crawlerNu
{
while (true)
{
if (CheckifShouldStop())
if (CheckIfShouldStop())
return;

CheckIfShouldPause();
Expand Down
9 changes: 6 additions & 3 deletions src/TumblThree/TumblThree.Applications/Crawler/TumblrLikedByCrawler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,9 @@ public override async Task IsBlogOnlineAsync()
}
catch (WebException webException)
{
if (webException.Status == WebExceptionStatus.RequestCanceled)
return;

Logger.Error("TumblrLikedByCrawler:IsBlogOnlineAsync:WebException {0}", webException);
shellService.ShowError(webException, Resources.BlogIsOffline, blog.Name);
blog.Online = false;
Expand All @@ -151,7 +154,7 @@ private long CreateStartPagination()
return dateTimeOffset.ToUnixTimeSeconds();
}

private bool CheckIfPagecountReached(int pageCount)
private bool CheckIfPageCountReached(int pageCount)
{
int numberOfPages = RangeToSequence(blog.DownloadPages).Count();
return pageCount >= numberOfPages;
Expand All @@ -164,7 +167,7 @@ private async Task<bool> CheckIfLoggedInAsync()
string document = await GetRequestAsync(blog.Url + "/page/1");
return !document.Contains("<div class=\"signup_view account login\"");
}
catch (WebException webException) when (webException.Response == null && webException.Status == WebExceptionStatus.RequestCanceled)
catch (WebException webException) when (webException.Status == WebExceptionStatus.RequestCanceled)
{
return true;
}
Expand All @@ -179,7 +182,7 @@ private async Task AddUrlsToDownloadListAsync(long pagination, int crawlerNumber
{
while (true)
{
if (CheckifShouldStop())
if (CheckIfShouldStop())
return;

CheckIfShouldPause();
Expand Down
2 changes: 1 addition & 1 deletion src/TumblThree/TumblThree.Applications/Crawler/TumblrSearchCrawler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ private async Task AddUrlsToDownloadListAsync(string response, int crawlerNumber
{
while (true)
{
if (CheckifShouldStop())
if (CheckIfShouldStop())
return;

CheckIfShouldPause();
Expand Down
4 changes: 2 additions & 2 deletions src/TumblThree/TumblThree.Applications/Crawler/TumblrTagSearchCrawler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ private async Task<bool> CheckIfLoggedInAsync()
string document = await GetTaggedSearchPageAsync(DateTimeOffset.Now.ToUnixTimeSeconds());
return !document.Contains("SearchResultsModel");
}
catch (WebException webException) when (webException.Response == null && webException.Status == WebExceptionStatus.RequestCanceled)
catch (WebException webException) when (webException.Status == WebExceptionStatus.RequestCanceled)
{
return true;
}
Expand Down Expand Up @@ -185,7 +185,7 @@ private async Task AddUrlsToDownloadListAsync(long pagination, long nextCrawlers
{
while (true)
{
if (CheckifShouldStop())
if (CheckIfShouldStop())
return;

CheckIfShouldPause();
Expand Down
2 changes: 1 addition & 1 deletion src/TumblThree/TumblThree.Applications/Downloader/AbstractDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ protected virtual async Task<bool> DownloadBinaryFileAsync(string fileLocation,
// The process cannot access the file because it is being used by another process.", HRESULT: -2147024864 == 0xFFFFFFFF80070020
return true;
}
catch (WebException webException) when ((webException.Response != null))
catch (WebException webException) when (webException.Response != null)
{
var webRespStatusCode = (int)((HttpWebResponse)webException.Response).StatusCode;
if (webRespStatusCode >= 400 && webRespStatusCode < 600
Expand Down

0 comments on commit 2622a52

Please sign in to comment.