From 9a301ebc5e16e1648d6b8dfe1ff7fdece38a6469 Mon Sep 17 00:00:00 2001 From: Andy <88590076+AAndyProgram@users.noreply.github.com> Date: Mon, 4 Apr 2022 03:00:22 +0300 Subject: [PATCH] 3.0.0.6 Added GoTo Start/End channels buttons Fixed saved Reddit posts downloading Fixed Reddit accessibility check Disabled main progress bar progress when downloading saved posts Added Date and Time for Stories and Tagged Photos --- Changelog.md | 11 ++++++++++ SCrawler/API/Base/DownDetector.vb | 27 +++++++++++++++--------- SCrawler/API/Instagram/UserData.vb | 30 +++++++++++++++++++++++---- SCrawler/API/Reddit/Channel.vb | 3 --- SCrawler/API/Reddit/SiteSettings.vb | 5 ++++- SCrawler/API/Reddit/UserData.vb | 2 +- SCrawler/Channels/ChannelViewForm.vb | 4 +++- SCrawler/Download/DownloadProgress.vb | 2 +- SCrawler/Download/TDownloader.vb | 4 +--- SCrawler/My Project/AssemblyInfo.vb | 4 ++-- 10 files changed, 66 insertions(+), 26 deletions(-) diff --git a/Changelog.md b/Changelog.md index c46b634..70895c6 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,14 @@ +# 3.0.0.6 + +- Added + - ```GoTo Start``` channels button + - ```GoTo End``` channels button +- Fixed + - In some cases, saved Reddit posts didn't fully download + - Incorrect Reddit accessibility check algorithm + - Incorrect behavior of the main progress bar when downloading saved posts + - (Issue #25) Date and Time not added for Stories and Tagged Photos + # 3.0.0.5 - Added diff --git a/SCrawler/API/Base/DownDetector.vb b/SCrawler/API/Base/DownDetector.vb index d3ed796..9079dfd 100644 --- a/SCrawler/API/Base/DownDetector.vb +++ b/SCrawler/API/Base/DownDetector.vb @@ -33,25 +33,32 @@ Namespace API.Base Friend Shared Function GetData(ByVal Site As String) As List(Of Data) Try Dim l As List(Of Data) = Nothing + Dim l2 As List(Of Data) = Nothing Using w As New WebClient Dim r$ = w.DownloadString($"https://downdetector.co.uk/status/{Site}/") If Not r.IsEmptyString Then l = FNF.RegexFields(Of Data)(r, {Params}, {1, 2}) If l.ListExists(2) Then - Dim lDate As Date = l(0).Date - Dim i% - Dim indx% = -1 - For i = 1 To l.Count - 1 - If l(i).Date < lDate Then indx = i : Exit For Else lDate = l(i).Date - Next - If indx >= 0 Then - For i = indx To 0 Step -1 : l.RemoveAt(i) : Next - End If l.Sort() + l2 = New List(Of Data) + Dim d As Data + Dim eDates As New List(Of Date) + Dim MaxValue As Func(Of Date, Integer) = Function(dd) (From ddd In l Where ddd.Date = dd Select ddd.Value).DefaultIfEmpty(0).Max + For i% = 0 To l.Count - 1 + If Not eDates.Contains(l(i).Date) Then + d = l(i) + d.Value = MaxValue(d.Date) + l2.Add(d) + eDates.Add(d.Date) + End If + Next + eDates.Clear() + l.Clear() + l2.Sort() End If End If End Using - Return l + Return l2 Catch ex As Exception Return ErrorsDescriber.Execute(EDP.SendInLog + EDP.ReturnValue, ex, $"[DownDetector.GetData({Site})]") End Try diff --git a/SCrawler/API/Instagram/UserData.vb b/SCrawler/API/Instagram/UserData.vb index d867aa2..9561b45 100644 --- a/SCrawler/API/Instagram/UserData.vb +++ b/SCrawler/API/Instagram/UserData.vb @@ -390,11 +390,30 @@ Namespace API.Instagram CreateMedia(node) End If End Sub - Private Sub ObtainMedia2(ByVal n As EContainer, ByVal PostID As String, Optional ByVal SpecialFolder As String = Nothing) + Private Sub ObtainMedia2(ByVal n As EContainer, ByVal PostID As String, Optional ByVal SpecialFolder As String = Nothing, + Optional ByVal DateObj As String = Nothing) Try Dim img As Predicate(Of EContainer) = Function(_img) Not _img.Name.IsEmptyString AndAlso _img.Name.StartsWith("image_versions") AndAlso _img.Count > 0 Dim vid As Predicate(Of EContainer) = Function(_vid) Not _vid.Name.IsEmptyString AndAlso _vid.Name.StartsWith("video_versions") AndAlso _vid.Count > 0 Dim ss As Func(Of EContainer, Sizes) = Function(_ss) New Sizes(_ss.Value("width"), _ss.Value("url")) + Dim mDate As Func(Of EContainer, String) = Function(ByVal elem As EContainer) As String + If elem.Contains("taken_at") Then + Return elem.Value("taken_at") + ElseIf elem.Contains("imported_taken_at") Then + Return elem.Value("imported_taken_at") + Else + Dim ev$ = elem.Value("device_timestamp") + If Not ev.IsEmptyString Then + If ev.Length > 10 Then + Return elem.Value("device_timestamp").Substring(0, 10) + Else + Return ev + End If + Else + Return String.Empty + End If + End If + End Function If n.Count > 0 Then Dim l As New List(Of Sizes) Dim d As EContainer @@ -408,6 +427,7 @@ Namespace API.Instagram Case 1 If n.Contains(img) Then t = n.Value("media_type").FromXML(Of Integer)(-1) + DateObj = mDate(n) If t >= 0 Then With n.ItemF({img, "candidates"}).XmlIfNothing If .Count > 0 Then @@ -415,7 +435,7 @@ Namespace API.Instagram l.ListAddList(.Select(ss), LNC) If l.Count > 0 Then l.Sort() - _TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, l.First.Data, PostID, Nothing, SpecialFolder), LNC) + _TempMediaList.ListAddValue(MediaFromData(UTypes.Picture, l.First.Data, PostID, DateObj, SpecialFolder), LNC) l.Clear() End If End If @@ -424,22 +444,24 @@ Namespace API.Instagram End If Case 2 If n.Contains(vid) Then + DateObj = mDate(n) With n.ItemF({vid}).XmlIfNothing If .Count > 0 Then l.Clear() l.ListAddList(.Select(ss), LNC) If l.Count > 0 Then l.Sort() - _TempMediaList.ListAddValue(MediaFromData(UTypes.Video, l.First.Data, PostID, Nothing, SpecialFolder), LNC) + _TempMediaList.ListAddValue(MediaFromData(UTypes.Video, l.First.Data, PostID, DateObj, SpecialFolder), LNC) l.Clear() End If End If End With End If Case 8 + DateObj = mDate(n) With n("carousel_media").XmlIfNothing If .Count > 0 Then - For Each d In .Self : ObtainMedia2(d, PostID, SpecialFolder) : Next + For Each d In .Self : ObtainMedia2(d, PostID, SpecialFolder, DateObj) : Next End If End With End Select diff --git a/SCrawler/API/Reddit/Channel.vb b/SCrawler/API/Reddit/Channel.vb index 381cfdb..584907d 100644 --- a/SCrawler/API/Reddit/Channel.vb +++ b/SCrawler/API/Reddit/Channel.vb @@ -84,7 +84,6 @@ Namespace API.Reddit Return Posts(Index) End Get End Property - Private ReadOnly Property Range As RangeSwitcher(Of UserPost) Friend Property ViewMode As View = View.New Implements IRedditView.ViewMode Friend Property ViewPeriod As Period = Period.All Implements IRedditView.ViewPeriod Friend Sub SetView(ByVal Options As IRedditView) Implements IRedditView.SetView @@ -219,7 +218,6 @@ Namespace API.Reddit Posts = New List(Of UserPost) PostsLatest = New List(Of UserPost) PostsNames = New List(Of String) - Range = New RangeSwitcher(Of UserPost)(Me) CountOfAddedUsers = New List(Of Integer) CountOfLoadedPostsPerSession = New List(Of Integer) ChannelExistentUserNames = New List(Of String) @@ -410,7 +408,6 @@ Namespace API.Reddit PostsNames.Clear() CountOfAddedUsers.Clear() CountOfLoadedPostsPerSession.Clear() - Range.Dispose() ChannelExistentUserNames.Clear() CachePath.Delete(SFO.Path, SFODelete.None, EDP.SendInLog) End If diff --git a/SCrawler/API/Reddit/SiteSettings.vb b/SCrawler/API/Reddit/SiteSettings.vb index 9fa0ee3..43d8c28 100644 --- a/SCrawler/API/Reddit/SiteSettings.vb +++ b/SCrawler/API/Reddit/SiteSettings.vb @@ -53,7 +53,10 @@ Namespace API.Reddit Case Download.Channel : Return New UserData With {.SaveToCache = False, .SkipExistsUsers = False, .AutoGetLimits = True} Case Download.SavedPosts Dim u As New UserData With {.IsSavedPosts = True} - DirectCast(u, UserDataBase).User = New UserInfo With {.Name = CStr(AConvert(Of String)(SavedPostsUserName.Value, String.Empty))} + DirectCast(u, UserDataBase).User = New UserInfo With { + .Name = CStr(AConvert(Of String)(SavedPostsUserName.Value, String.Empty)), + .IsChannel = True + } Return u End Select Return Nothing diff --git a/SCrawler/API/Reddit/UserData.vb b/SCrawler/API/Reddit/UserData.vb index 067fad6..04d9c67 100644 --- a/SCrawler/API/Reddit/UserData.vb +++ b/SCrawler/API/Reddit/UserData.vb @@ -340,7 +340,7 @@ Namespace API.Reddit If s.Contains("created") Then PostDate = s("created").Value Else PostDate = String.Empty _UserID = s.Value("author") - If SkipExistsUsers AndAlso _ExistsUsersNames.Count > 0 AndAlso + If Not IsSavedPosts AndAlso SkipExistsUsers AndAlso _ExistsUsersNames.Count > 0 AndAlso Not _UserID.IsEmptyString AndAlso _ExistsUsersNames.Contains(_UserID) Then If Not IsSavedPosts AndAlso Not ChannelInfo Is Nothing Then _ ChannelInfo.ChannelExistentUserNames.ListAddValue(_UserID, LNC) diff --git a/SCrawler/Channels/ChannelViewForm.vb b/SCrawler/Channels/ChannelViewForm.vb index f712353..66b4694 100644 --- a/SCrawler/Channels/ChannelViewForm.vb +++ b/SCrawler/Channels/ChannelViewForm.vb @@ -201,11 +201,12 @@ Friend Class ChannelViewForm : Implements IChannelLimits MyRange = New RangeSwitcher(Of UserPost) With {.Selector = SelectorExpression} With MyRange .Limit = ImagesInRow * ImagesRows - .InsertButtons(ToolbarTOP, {RButton.Previous, RButton.Next}, 5) + .InsertButtons(ToolbarTOP,, 5) .SetButtonKey(RButton.Previous, Keys.F2) .SetButtonKey(RButton.Next, Keys.F3) .BindForm(Me) .LabelNumbersProvider = CProvider + .LabelShowAbsolutIndexes = False .UpdateControls() End With AddHandler Settings.ChannelsImagesColumns.OnValueChanged, AddressOf ImagesCountChanged @@ -215,6 +216,7 @@ Friend Class ChannelViewForm : Implements IChannelLimits MyDefs.MyViewInitialize(Me, Settings.Design) RefillChannels(Settings.LatestSelectedChannel.Value) ChangeComboIndex(0) + MyRange.LabelText = String.Empty CMB_CHANNELS_ActionOnCheckedChange(CMB_CHANNELS.Checked) With LIST_POSTS Dim s As Size = GetImageSize() diff --git a/SCrawler/Download/DownloadProgress.vb b/SCrawler/Download/DownloadProgress.vb index 940c515..9692176 100644 --- a/SCrawler/Download/DownloadProgress.vb +++ b/SCrawler/Download/DownloadProgress.vb @@ -156,7 +156,7 @@ Namespace DownloadObjects RaiseEvent OnTotalCountChange() End Sub Private Sub JobProgress_OnProgressChange(ByVal Source As IMyProgress, ByVal Index As Integer) - MainProgress.Perform() + If Not Job.Type = Download.SavedPosts Then MainProgress.Perform() End Sub #End Region #Region "IDisposable Support" diff --git a/SCrawler/Download/TDownloader.vb b/SCrawler/Download/TDownloader.vb index c5de187..2e4881a 100644 --- a/SCrawler/Download/TDownloader.vb +++ b/SCrawler/Download/TDownloader.vb @@ -254,9 +254,7 @@ Namespace DownloadObjects Dim SiteChecked As Boolean = False Do While _Job.Count > 0 _Job.ThrowIfCancellationRequested() - If Not SiteChecked Then - If Not _Job.Available Then Exit Sub Else SiteChecked = True : Continue Do - End If + If Not SiteChecked Then _Job.Available() : SiteChecked = True : Continue Do UpdateJobsLabel() DownloadData(_Job, _Job.Token) _Job.ThrowIfCancellationRequested() diff --git a/SCrawler/My Project/AssemblyInfo.vb b/SCrawler/My Project/AssemblyInfo.vb index a29592f..1cc927e 100644 --- a/SCrawler/My Project/AssemblyInfo.vb +++ b/SCrawler/My Project/AssemblyInfo.vb @@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices ' by using the '*' as shown below: ' - - + +