From ca384e54d6bf69995b7e5a81f29dfb20f6e4c002 Mon Sep 17 00:00:00 2001 From: Andy <88590076+AAndyProgram@users.noreply.github.com> Date: Wed, 29 May 2024 02:45:37 +0300 Subject: [PATCH] 2024.5.29.0 YT Trim urls to get rid of 'cr' & 'lf' Get the correct 'music' url for 'url' files SCrawler Remove the no longer needed 'MainFrameObj.UpdateLogButton' from the classes ProfileSaved: swap the 'ReadyToDownload' and 'Available' checks; remove the 'DownloadStarted' and 'DownloadDone' calls because they are called in the root function API.Instagram: improve availability checking API.Twitter: fix deleting user directory when redownloading missing posts AutoDownloader: improve statuses; move the check thread to the scheduler; add highlighting of scheduler plans (working, stopped, pending, etc.); replace 'ListBox' with 'ListView'; highlight undownloaded plans in gray --- SCrawler.YouTube/Base/YouTubeFunctions.vb | 6 +- SCrawler.YouTube/Declarations.vb | 6 + SCrawler.YouTube/Downloader/VideoListForm.vb | 2 +- .../Objects/YouTubeMediaContainerBase.vb | 2 +- SCrawler/API/Base/ProfileSaved.vb | 12 +- SCrawler/API/Instagram/SiteSettings.vb | 35 ++-- SCrawler/API/Twitter/UserData.vb | 9 +- .../Download/ActiveDownloadingProgress.vb | 1 - .../Download/Automation/AutoDownloader.vb | 159 ++++++++++++------ SCrawler/Download/Automation/Scheduler.vb | 106 +++++++++--- .../SchedulerEditorForm.Designer.vb | 22 ++- .../Automation/SchedulerEditorForm.vb | 127 +++++++++++--- SCrawler/Download/Feed/DownloadFeedForm.vb | 3 - SCrawler/Download/Feed/FeedSpecial.vb | 1 - .../Download/Feed/FeedSpecialCollection.vb | 2 - .../STDownloader/DownloaderUrlForm.vb | 1 - .../STDownloader/DownloaderUrlsArrForm.vb | 1 - .../STDownloader/VideoDownloaderForm.vb | 4 - SCrawler/Download/TDownloader.vb | 39 +++-- .../Editors/GlobalSettingsForm.Designer.vb | 26 ++- SCrawler/Editors/GlobalSettingsForm.vb | 2 + SCrawler/Editors/UsersInfoForm.vb | 1 - SCrawler/MainFrame.vb | 4 - SCrawler/MainFrameObjects.vb | 12 +- .../Hosts/SettingsHostCollection.vb | 1 - SCrawler/SettingsCLS.vb | 4 + 26 files changed, 413 insertions(+), 175 deletions(-) diff --git a/SCrawler.YouTube/Base/YouTubeFunctions.vb b/SCrawler.YouTube/Base/YouTubeFunctions.vb index 040b53b..1568345 100644 --- a/SCrawler.YouTube/Base/YouTubeFunctions.vb +++ b/SCrawler.YouTube/Base/YouTubeFunctions.vb @@ -22,6 +22,7 @@ Namespace API.YouTube.Base End Sub Public Shared Function StandardizeURL(ByVal URL As String) As String Try + URL = URL.StringTrim Dim isMusic As Boolean = False, isShorts As Boolean = False If Info_GetUrlType(URL, isMusic, isShorts) = YouTubeMediaType.Single Then If Not isMusic And Not isShorts Then @@ -45,6 +46,7 @@ Namespace API.YouTube.Base End Function Public Shared Function StandardizeURL_Channel(ByVal URL As String, Optional ByVal Process As Boolean = True) As String Try + URL = URL.StringTrim Dim ct As YouTubeChannelTab = YouTubeChannelTab.All Dim isMusic As Boolean = False If Process AndAlso Info_GetUrlType(URL, isMusic,,,, ct) = YouTubeMediaType.Channel AndAlso Not isMusic Then @@ -72,6 +74,7 @@ Namespace API.YouTube.Base Public Shared Function Info_GetUrlType(ByVal URL As String, Optional ByRef IsMusic As Boolean = False, Optional ByRef IsShorts As Boolean = False, Optional ByRef IsChannelUser As Boolean = False, Optional ByRef Id As String = Nothing, Optional ByRef ChannelOptions As YouTubeChannelTab = YouTubeChannelTab.All) As YouTubeMediaType + URL = URL.StringTrim If Not URL.IsEmptyString Then IsMusic = URL.Contains("music.youtube.com") IsChannelUser = False @@ -118,6 +121,7 @@ Namespace API.YouTube.Base Optional ByVal Token As Threading.CancellationToken = Nothing, Optional ByVal Progress As IMyProgress = Nothing, Optional ByVal DateAfter As Date? = Nothing, Optional ByVal DateBefore As Date? = Nothing, Optional ByVal ChannelOption As YouTubeChannelTab? = Nothing, Optional ByVal UrlAsIs As Boolean = False) As IYouTubeMediaContainer + URL = URL.StringTrim If URL.IsEmptyString Then Throw New ArgumentNullException("URL", "URL cannot be null") If Not MyYouTubeSettings.YTDLP.Value.Exists Then Throw New IO.FileNotFoundException("Path to 'yt-dlp.exe' not set or program not found at destination", MyYouTubeSettings.YTDLP.Value.ToString) Dim urlOrig$ = URL @@ -162,7 +166,7 @@ Namespace API.YouTube.Base If result Then container.Parse(Nothing, _CachePathDefault, isMusic, Token, Progress) - If Not container.HasError Then container.URL = URL : container.IsShorts = isShorts : Return container + If Not container.HasError Then container.URL = URL.ToMusicUrl(isMusic) : container.IsShorts = isShorts : Return container End If container.Dispose() End If diff --git a/SCrawler.YouTube/Declarations.vb b/SCrawler.YouTube/Declarations.vb index d092735..920dfeb 100644 --- a/SCrawler.YouTube/Declarations.vb +++ b/SCrawler.YouTube/Declarations.vb @@ -6,6 +6,7 @@ ' ' This program is distributed in the hope that it will be useful, ' but WITHOUT ANY WARRANTY +Imports System.Runtime.CompilerServices Imports PersonalUtilities.Tools Imports PersonalUtilities.Forms.Toolbars Imports PersonalUtilities.Functions.RegularExpressions @@ -56,6 +57,11 @@ Namespace API.YouTube Friend ReadOnly TitleHtmlConverter As Func(Of String, String) = Function(Input) Input.StringRemoveWinForbiddenSymbols().StringTrim() Friend ReadOnly ProgressProvider As IMyProgressNumberProvider = MyProgressNumberProvider.Percentage Public ReadOnly TrueUrlRegEx As RParams = RParams.DM(Base.YouTubeFunctions.TrueUrlPattern, 0, EDP.ReturnValue) + Friend ReadOnly MusicUrlApply As RParams = RParams.DMS("https://([w\.]*)youtube.com.+", 1, RegexReturn.Replace, EDP.ReturnValue, + CType(Function(input$) "music.", Func(Of String, String)), String.Empty) + Friend Function ToMusicUrl(ByVal URL As String, ByVal IsMusic As Boolean) As String + Try : Return If(IsMusic And Not URL.IsEmptyString, CStr(RegexReplace(URL, MusicUrlApply)).IfNullOrEmpty(URL), URL) : Catch : Return URL : End Try + End Function Friend Function CleanFileName(ByVal f As SFile) As SFile If Not f.IsEmptyString And Not f.Name.IsEmptyString Then Dim ff As SFile = f diff --git a/SCrawler.YouTube/Downloader/VideoListForm.vb b/SCrawler.YouTube/Downloader/VideoListForm.vb index b14547e..2fc0226 100644 --- a/SCrawler.YouTube/Downloader/VideoListForm.vb +++ b/SCrawler.YouTube/Downloader/VideoListForm.vb @@ -247,7 +247,7 @@ Namespace DownloadObjects.STDownloader Dim useCookiesParse As Boolean? = Nothing If useCookies Then useCookiesParse = True Dim standardizeUrls As Boolean = MyYouTubeSettings.StandardizeURLs - Dim standardize As Func(Of String, String) = Function(input) If(standardizeUrls, YouTubeFunctions.StandardizeURL(input), input) + Dim standardize As Func(Of String, String) = Function(input) If(standardizeUrls, YouTubeFunctions.StandardizeURL(input), input.StringTrim) Dim c As IYouTubeMediaContainer = Nothing Dim url$ = String.Empty diff --git a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb index a0a917a..ad6a361 100644 --- a/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb +++ b/SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb @@ -1544,7 +1544,7 @@ Namespace API.YouTube.Objects ID = .Value("id") Title = TitleHtmlConverter.Invoke(.Value("title")) Description = .Value("description") - URL = .Value("webpage_url") + URL = .Value("webpage_url").ToMusicUrl(IsMusic) PlaylistID = .Value("playlist_id") PlaylistCount = .Value("n_entries").IfNullOrEmpty(.Value("playlist_count")).FromXML(Of Integer)(0) diff --git a/SCrawler/API/Base/ProfileSaved.vb b/SCrawler/API/Base/ProfileSaved.vb index ffa067a..c14f1e1 100644 --- a/SCrawler/API/Base/ProfileSaved.vb +++ b/SCrawler/API/Base/ProfileSaved.vb @@ -54,9 +54,8 @@ Namespace API.Base Dim aStr$ = String.Empty If Count > 1 Then aStr = $" ({Number}/{Count})" Try - If Host.Source.ReadyToDownload(PDownload.SavedPosts) Then - If Host.Available(PDownload.SavedPosts, Multiple Or Count > 1) Then - Host.DownloadStarted(PDownload.SavedPosts) + If Host.Available(PDownload.SavedPosts, Multiple Or Count > 1) Then + If Host.Source.ReadyToDownload(PDownload.SavedPosts) Then If Count > 1 Then Progress.Information = $"{Host.Name} - {Host.AccountName.IfNullOrEmpty(SettingsHost.NameAccountNameDefault)}" Using user As IUserData = Host.GetInstance(PDownload.SavedPosts, Nothing, False, False) If Not user Is Nothing Then @@ -83,11 +82,11 @@ Namespace API.Base End Using Else _Unavailable += 1 - Progress.InformationTemporary = $"Host [{Host.Name}{aStr}] is unavailable" + Progress.InformationTemporary = $"Host [{Host.Name}{aStr}] is not ready" End If Else _NotReady += 1 - Progress.InformationTemporary = $"Host [{Host.Name}{aStr}] is not ready" + Progress.InformationTemporary = $"Host [{Host.Name}{aStr}] is unavailable" End If Catch oex As OperationCanceledException When Token.IsCancellationRequested _ErrorCount += 1 @@ -96,9 +95,6 @@ Namespace API.Base _ErrorCount += 1 Progress.InformationTemporary = $"{Host.Name}{aStr} downloading error" ErrorsDescriber.Execute(EDP.SendToLog, ex, $"[API.Base.ProfileSaved.Download({Host.Key}{aStr})]") - Finally - Host.DownloadDone(PDownload.SavedPosts) - MainFrameObj.UpdateLogButton() End Try End Sub End Class diff --git a/SCrawler/API/Instagram/SiteSettings.vb b/SCrawler/API/Instagram/SiteSettings.vb index ac8fa19..4346d6b 100644 --- a/SCrawler/API/Instagram/SiteSettings.vb +++ b/SCrawler/API/Instagram/SiteSettings.vb @@ -480,8 +480,24 @@ Namespace API.Instagram Private ____AvailableSilent As Boolean = True Private ____AvailableChecked As Boolean = False Private ____AvailableResult As Boolean = False + Private Sub ResetDownloadOptions() + If ActiveJobs < 1 Then + ____DownloadStarted = False + ____AvailableRequested = False + ____AvailableChecked = False + ____AvailableSilent = True + ____AvailableResult = False + If ActiveSessionRequestsExists Then RefreshMyLastRequests(Now) + ActiveSessionRequestsExists = False + _NextWNM = UserData.WNM.Notify + _NextTagged = True + SkipUntilNextSession = False + AvailableText = String.Empty + ActiveJobs = 0 + End If + End Sub Friend Overrides Function Available(ByVal What As Download, ByVal Silent As Boolean) As Boolean - If MyBase.Available(What, Silent) Then + If MyBase.Available(What, Silent) And ActiveJobs < 2 Then If CInt(DownDetectorValue.Value) >= 0 Then If ____DownloadStarted Then ____AvailableRequested = True @@ -551,10 +567,9 @@ Namespace API.Instagram Private _NextWNM As UserData.WNM = UserData.WNM.Notify Private _NextTagged As Boolean = True Friend Overrides Sub DownloadStarted(ByVal What As Download) - If ActiveJobs = 0 Then ActiveSessionRequestsExists = False + ResetDownloadOptions() ActiveJobs += 1 - If What = Download.Main Then ____DownloadStarted = True - If ActiveJobs = 1 Then ActiveSessionDate = Now + If ActiveJobs = 1 Then ____DownloadStarted = True : ActiveSessionDate = Now If Not HH_IG_WWW_CLAIM_IS_ZERO AndAlso ( (CBool(HH_IG_WWW_CLAIM_USE_DEFAULT_ALGO.Value) AndAlso MyLastRequestsDate.AddMinutes(HH_IG_WWW_CLAIM_UPDATE_INTERVAL.Value) < Now) Or @@ -594,18 +609,8 @@ Namespace API.Instagram End With End Sub Friend Overrides Sub DownloadDone(ByVal What As Download) - _NextWNM = UserData.WNM.Notify - _NextTagged = True - If ActiveSessionRequestsExists Then RefreshMyLastRequests(Now) ActiveJobs -= 1 - SkipUntilNextSession = False - If What = Download.Main Then ____DownloadStarted = False - If ActiveJobs = 0 Then - ____AvailableRequested = False - ____AvailableChecked = False - ____AvailableSilent = True - ____AvailableResult = False - End If + ResetDownloadOptions() End Sub #End Region #Region "Settings" diff --git a/SCrawler/API/Twitter/UserData.vb b/SCrawler/API/Twitter/UserData.vb index 44ad357..1a6eafc 100644 --- a/SCrawler/API/Twitter/UserData.vb +++ b/SCrawler/API/Twitter/UserData.vb @@ -752,14 +752,7 @@ Namespace API.Twitter Dim lim% Dim specFolder$ = IIf(_ReparseLikes, "Likes", String.Empty) ResetFileNameProvider() - If IsSingleObjectDownload Then - cache = Settings.Cache - ElseIf _ReparseLikes Then - cache = CreateCache() - Else - cache = New CacheKeeper(DownloadContentDefault_GetRootDir.CSFilePS) - cache.CacheDeleteError = CacheDeletionError(cache) - End If + cache = If(IsSingleObjectDownload, Settings.Cache, CreateCache()) If _ReparseLikes Then lim = LikesPosts.Count Else lim = _ContentList.Count ProgressPre.ChangeMax(lim) For i = 0 To lim - 1 diff --git a/SCrawler/Download/ActiveDownloadingProgress.vb b/SCrawler/Download/ActiveDownloadingProgress.vb index 941630a..682837a 100644 --- a/SCrawler/Download/ActiveDownloadingProgress.vb +++ b/SCrawler/Download/ActiveDownloadingProgress.vb @@ -58,7 +58,6 @@ Namespace DownloadObjects End If Catch ex As Exception ErrorsDescriber.Execute(EDP.SendToLog, ex, "Change 'ActiveDownloadingProgress' size") - MainFrameObj.UpdateLogButton() End Try End Sub Private Sub Downloader_Reconfigured() diff --git a/SCrawler/Download/Automation/AutoDownloader.vb b/SCrawler/Download/Automation/AutoDownloader.vb index 32ee52f..38f2330 100644 --- a/SCrawler/Download/Automation/AutoDownloader.vb +++ b/SCrawler/Download/Automation/AutoDownloader.vb @@ -16,6 +16,7 @@ Imports PersonalUtilities.Tools.Notifications Namespace DownloadObjects Friend Class AutoDownloader : Inherits GroupParameters : Implements IIndexable, IEContainerProvider, IComparable(Of AutoDownloader) Friend Event PauseChanged(ByVal Value As PauseModes) + Friend Event PlanChanged As Scheduler.PlanChangedEventHandler Friend Enum Modes As Integer None = 0 Specified = 3 @@ -199,6 +200,52 @@ Namespace DownloadObjects Private Const Name_ShowSimpleNotification As String = "ShowSimpleNotification" #End Region #Region "Declarations" + Private _LVIState As Integer = 0 + Friend Function LVIStateChanged() As Boolean + Dim ____LVIState% + Dim result As Boolean + If Downloading Then + ____LVIState = 1 + ElseIf DownloadReady(True, True) Or ForceStartRequested Then + ____LVIState = 2 + ElseIf Not Working And Not IsManual Then + ____LVIState = 3 + ElseIf IsPaused Then + ____LVIState = 4 + ElseIf Settings.AutomationBrushUndownloadedPlansMinutes.Value > 0 AndAlso + LastDownloadDate.AddMinutes(Settings.AutomationBrushUndownloadedPlansMinutes.Value) < Now Then + ____LVIState = 5 + Else + ____LVIState = 0 + End If + result = Not _LVIState = ____LVIState + _LVIState = ____LVIState + Return result + End Function + Friend ReadOnly Property LVI As ListViewItem + Get + Dim l As New ListViewItem(ToString) + If Downloading Then + l.BackColor = MyColor.OkBack + l.ForeColor = MyColor.OkFore + ElseIf DownloadReady(True, True) Or ForceStartRequested Then + l.BackColor = MyColor.EditBack + l.ForeColor = MyColor.EditFore + ElseIf Not Working And Not IsManual Then + l.BackColor = MyColor.DeleteBack + l.ForeColor = MyColor.DeleteFore + ElseIf Working And IsPaused And Not IsManual Then + l.BackColor = MyColor.UpdateBack + l.ForeColor = MyColor.UpdateFore + ElseIf Settings.AutomationBrushUndownloadedPlansMinutes.Value > 0 AndAlso + LastDownloadDate.AddMinutes(Settings.AutomationBrushUndownloadedPlansMinutes.Value) < Now Then + l.BackColor = Color.FromArgb(224, 224, 224) + l.ForeColor = Color.FromArgb(64, 64, 64) + End If + LVIStateChanged() + Return l + End Get + End Property Friend Property Source As Scheduler Private _Mode As Modes = Modes.None Friend Property Mode As Modes @@ -267,11 +314,15 @@ Namespace DownloadObjects End Property Private Function GetWorkingState() As String Dim OutStr$ - If Working Then - If StartupDelay > 0 And _StartTime.AddMinutes(StartupDelay) > Now Then - OutStr = $"delayed until {_StartTime.AddMinutes(StartupDelay).ToStringDate(ADateTime.Formats.BaseDateTime)}" + If Working And Not IsManual Then + If _Downloading Then + OutStr = "downloading" + ElseIf _ForceStartRequested Then + OutStr = "force start pending" ElseIf _StopRequested Then OutStr = "stopping" + ElseIf StartupDelay > 0 And _StartTime.AddMinutes(StartupDelay) > Now Then + OutStr = $"delayed ({StartupDelay}) until {_StartTime.AddMinutes(StartupDelay).ToStringDate(ADateTime.Formats.BaseDateTime)}" Else OutStr = "working" End If @@ -282,14 +333,26 @@ Namespace DownloadObjects Case Else : OutStr &= $", paused ([{Pause}] until {AConvert(Of String)(_PauseValue, ADateTime.Formats.BaseDateTime, "?")})" End Select End If + ElseIf IsManual Then + If _Downloading Then + OutStr = "downloading" + ElseIf _ForceStartRequested Then + OutStr = "force start pending" + Else + OutStr = String.Empty + End If Else OutStr = "stopped" End If Return OutStr End Function + Private Function GetWorkingState_Manual() As String + Dim OutStr$ = GetWorkingState() + If Not OutStr.IsEmptyString Then Return $" ({OutStr})" Else Return String.Empty + End Function Public Overrides Function ToString() As String If IsManual Then - Return $"{Name} (manual): last download date: {GetLastDateString()}" + Return $"{Name} (manual{GetWorkingState_Manual()}): last download date: {GetLastDateString()}" Else Return $"{Name} ({GetWorkingState()}): last download date: {GetLastDateString()}; next run: {GetNextDateString()}" End If @@ -316,6 +379,7 @@ Namespace DownloadObjects Friend Sub New(ByVal x As EContainer) Me.New Initialization = True + 'URGENT: replace this line Dim m% = x.Value(Name_Mode).FromXML(Of Integer)(Modes.None) If m = 1 Or m = 2 Then m = Modes.Specified Mode = m @@ -395,10 +459,10 @@ Namespace DownloadObjects End Function #End Region #Region "Execution" - Private AThread As Thread + Private _Working As Boolean = False Friend ReadOnly Property Working As Boolean Get - Return If(AThread?.IsAlive, False) + Return _Working End Get End Property Private _StartTime As Date = Now @@ -406,11 +470,8 @@ Namespace DownloadObjects If Not IsManual Or Force Then If Init Then _StartTime = Now _IsNewPlan = False - If Not Working And Not Mode = Modes.None Then - AThread = New Thread(New ThreadStart(AddressOf Checker)) - AThread.SetApartmentState(ApartmentState.MTA) - AThread.Start() - End If + If Not Working And Not Mode = Modes.None Then _Working = True + RaiseEvent PlanChanged(Me) End If End Sub Private _StopRequested As Boolean = False @@ -434,6 +495,7 @@ Namespace DownloadObjects Case Else : _PauseValue = Nothing End Select RaiseEvent PauseChanged(p) + RaiseEvent PlanChanged(Me) End Set End Property Private ReadOnly Property IsPaused As Boolean @@ -457,7 +519,10 @@ Namespace DownloadObjects End Get End Property Friend Sub [Stop]() - If Working Then _StopRequested = True + If Working Then + If Downloading Then _StopRequested = True Else _Working = False + RaiseEvent PlanChanged(Me) + End If End Sub Friend Overloads Sub Skip() If LastDownloadDate.AddMinutes(Timer) <= Now Then @@ -465,65 +530,59 @@ Namespace DownloadObjects Else _LastDownloadDateSkip = LastDownloadDate.AddMinutes(Timer) End If + RaiseEvent PlanChanged(Me) End Sub Friend Overloads Sub Skip(ByVal Minutes As Integer) _LastDownloadDateSkip = If(_LastDownloadDateSkip, Now).AddMinutes(Minutes) + RaiseEvent PlanChanged(Me) End Sub Friend Overloads Sub Skip(ByVal ToDate As Date) _LastDownloadDateSkip = ToDate + RaiseEvent PlanChanged(Me) End Sub Friend Sub SkipReset() _LastDownloadDateSkip = Nothing + RaiseEvent PlanChanged(Me) End Sub Friend Sub ForceStart() _ForceStartRequested = True If IsManual Then Start(False, True) + RaiseEvent PlanChanged(Me) End Sub Private _ForceStartRequested As Boolean = False - Private _SpecialDelayUse As Boolean = False - Private _SpecialDelayTime As Date? = Nothing - Private Sub Checker() - Try - Dim _StartDownload As Boolean - While (Not _StopRequested Or Downloader.Working) And Not Mode = Modes.None - If ((IsManual And _ForceStartRequested) Or (NextExecutionDate < Now And Not IsPaused) Or _ForceStartRequested) And - Not _StopRequested And Not Mode = Modes.None Then - If Downloader.Working Then - _SpecialDelayUse = True - Else - If _SpecialDelayUse And Not _SpecialDelayTime.HasValue Then _SpecialDelayTime = Now.AddSeconds(10) - If Not _SpecialDelayUse OrElse (_SpecialDelayTime.HasValue AndAlso _SpecialDelayTime.Value < Now) Then - _SpecialDelayUse = False - _SpecialDelayTime = Nothing - _StartDownload = False - If Settings.Automation.Count = 1 Or _ForceStartRequested Or Index = -1 Then - _StartDownload = True - Else - _StartDownload = NextExecutionDate.AddMilliseconds(1000 * (Index + 1)).Ticks <= Now.Ticks - End If - If _StartDownload Then - Download() - If IsManual Then Exit While - End If - End If - End If + Friend ReadOnly Property ForceStartRequested As Boolean + Get + Return _ForceStartRequested + End Get + End Property + Friend ReadOnly Property DownloadReady(Optional ByVal IgnorePause As Boolean = False, Optional ByVal IgnoreDownloaderWorking As Boolean = False) As Boolean + Get + If _StopRequested Then _Working = False + Return (Working Or IsManual) And ((IsManual And _ForceStartRequested) Or (Not IsManual And NextExecutionDate < Now And (Not IsPaused Or IgnorePause)) Or _ForceStartRequested) And + Not _StopRequested And Not Mode = Modes.None And (Not Downloader.Working Or IgnoreDownloaderWorking) + End Get + End Property + Friend ReadOnly Property NextDate As Date? + Get + If Not _StopRequested And Not Mode = Modes.None Then + If IsManual Or _ForceStartRequested Then + Return Now.AddYears(-10) + ElseIf Not IsPaused And Not IsManual And Working Then + Return NextExecutionDate End If - Thread.Sleep(500) - End While - Catch ex As Exception - ErrorsDescriber.Execute(EDP.SendToLog, ex, "[AutoDownloader.Checker]") - Finally - _StopRequested = False - End Try - End Sub + End If + Return Nothing + End Get + End Property Private _Downloading As Boolean = False Friend ReadOnly Property Downloading As Boolean Get Return _Downloading End Get End Property - Private Sub Download() + Friend Sub Download() _Downloading = True + RaiseEvent PlanChanged(Me) Dim Keys As New List(Of String) Try Dim users As New List(Of IUserData) @@ -589,7 +648,7 @@ Namespace DownloadObjects While .Working Or .Count > 0 : notify.Invoke() : Thread.Sleep(200) : End While .AutoDownloaderWorking = False notify.Invoke - If simple And DownloadedUsersCount + DownloadedSubscriptionsCount > 0 Then + If simple And (DownloadedUsersCount + DownloadedSubscriptionsCount) > 0 Then Dim msg$ = String.Empty If DownloadedUsersCount > 0 Then msg = $"{DownloadedUsersCount} user(s) " If DownloadedSubscriptionsCount > 0 Then msg &= $"{IIf(DownloadedUsersCount > 0, "and ", String.Empty)}{DownloadedSubscriptionsCount} subscription(s) " @@ -607,6 +666,8 @@ Namespace DownloadObjects _Downloading = False _ForceStartRequested = False _LastDownloadDateSkip = Nothing + If _StopRequested Then _Working = False + RaiseEvent PlanChanged(Me) End Try End Sub Private Sub ShowNotification(ByVal u As IUserData) diff --git a/SCrawler/Download/Automation/Scheduler.vb b/SCrawler/Download/Automation/Scheduler.vb index 9ce7b85..a476c57 100644 --- a/SCrawler/Download/Automation/Scheduler.vb +++ b/SCrawler/Download/Automation/Scheduler.vb @@ -14,15 +14,19 @@ Imports PauseModes = SCrawler.DownloadObjects.AutoDownloader.PauseModes Namespace DownloadObjects Friend Class Scheduler : Implements IEnumerable(Of AutoDownloader), IMyEnumerator(Of AutoDownloader), IDisposable Friend Const Name_Plan As String = "Plan" + Friend Delegate Sub PlanChangedEventHandler(ByVal Plan As AutoDownloader) Friend Event PauseChanged As AutoDownloader.PauseChangedEventHandler Private Sub OnPauseChanged(ByVal Value As PauseModes) RaiseEvent PauseChanged(Pause) End Sub + Friend Event PlanChanged As PlanChangedEventHandler + Private Sub OnPlanChanged(ByVal Plan As AutoDownloader) + Try : RaiseEvent PlanChanged(Plan) : Catch : End Try + End Sub Private ReadOnly Plans As List(Of AutoDownloader) Friend Const FileNameDefault As String = "AutoDownload" Friend ReadOnly FileDefault As SFile = $"{SettingsFolderName}\{FileNameDefault}.xml" Friend File As SFile = Nothing - Private ReadOnly PlanWorking As Predicate(Of AutoDownloader) = Function(Plan) Plan.Working Private ReadOnly PlanDownloading As Predicate(Of AutoDownloader) = Function(Plan) Plan.Downloading Private ReadOnly PlansWaiter As Action(Of Predicate(Of AutoDownloader)) = Sub(ByVal Predicate As Predicate(Of AutoDownloader)) While Plans.Exists(Predicate) : Thread.Sleep(200) : End While @@ -69,6 +73,7 @@ Namespace DownloadObjects Friend Sub Add(ByVal Plan As AutoDownloader) Plan.Source = Me AddHandler Plan.PauseChanged, AddressOf OnPauseChanged + AddHandler Plan.PlanChanged, AddressOf OnPlanChanged Plans.Add(Plan) Plans.ListReindex Update() @@ -77,9 +82,9 @@ Namespace DownloadObjects If Index.ValueBetween(0, Count - 1) Then With Plans(Index) .Stop() - If .Working Then + If .Downloading Then Await Task.Run(Sub() - While .Working : Thread.Sleep(510) : End While + While .Downloading : Thread.Sleep(510) : End While End Sub) End If .Dispose() @@ -103,10 +108,10 @@ Namespace DownloadObjects End Try End Sub Friend Function Reset(ByVal f As SFile, ByVal IsInit As Boolean) As Boolean - If Plans.Count > 0 Then - If Not Plans.Exists(PlanWorking) Then + If Count > 0 Then + If Not Plans.Exists(PlanDownloading) Then Pause = PauseModes.Unlimited - If Plans.Exists(PlanWorking) Then + If Plans.Exists(PlanDownloading) Then MsgBoxE({$"Some plans are already being worked.{vbCr}Wait for the plans to complete their work and try again.", "Change scheduler"}, vbCritical) Pause = PauseModes.Unlimited @@ -114,6 +119,7 @@ Namespace DownloadObjects End If End If [Stop]() + While Working : Thread.Sleep(200) : End While If _UpdateRequired Then Update() Plans.ListClearDispose(,, EDP.LogMessageValue) End If @@ -127,11 +133,12 @@ Namespace DownloadObjects Plans.Add(New AutoDownloader(x)) End If End Using - If Plans.Count > 0 Then Plans.ForEach(Sub(ByVal p As AutoDownloader) - p.Source = Me - If Not IsInit Then p.Pause = PauseModes.Unlimited - AddHandler p.PauseChanged, AddressOf OnPauseChanged - End Sub) : Plans.ListReindex + If Count > 0 Then Plans.ForEach(Sub(ByVal p As AutoDownloader) + p.Source = Me + If Not IsInit Then p.Pause = PauseModes.Unlimited + AddHandler p.PauseChanged, AddressOf OnPauseChanged + AddHandler p.PlanChanged, AddressOf OnPlanChanged + End Sub) : Plans.ListReindex End If Return True End Function @@ -163,38 +170,94 @@ Namespace DownloadObjects End Sub #End Region #Region "Execution" + Private AThread As Thread = Nothing + Private _StopRequested As Boolean = False + Friend ReadOnly Property Working As Boolean + Get + Return If(AThread?.IsAlive, False) + End Get + End Property Friend Async Function Start(ByVal Init As Boolean) As Task Try + _StopRequested = False Await Task.Run(Sub() Dim r% = 0 Do r += 1 Try - If Count > 0 Then - If Plans.Exists(PlanDownloading) Then PlansWaiter(PlanDownloading) - For Each Plan In Plans - Plan.Start(Init) - PlansWaiter(PlanDownloading) - Thread.Sleep(1000) - Next - End If + If Count > 0 Then PlansWaiter(PlanDownloading) : Plans.ForEach(Sub(p) p.Start(Init)) Exit Do Catch io_ex As InvalidOperationException 'Collection was modified; enumeration operation may not execute End Try Loop While r < 10 End Sub) + If Not Working Then + AThread = New Thread(New ThreadStart(AddressOf Checker)) + AThread.SetApartmentState(ApartmentState.MTA) + AThread.Start() + End If Catch ex As Exception If Init Then ErrorsDescriber.Execute(EDP.SendToLog, ex, "Start automation") - MainFrameObj.UpdateLogButton() Else Throw ex End If End Try End Function Friend Sub [Stop]() + If Working Then _StopRequested = True If Count > 0 Then Plans.ForEach(Sub(p) p.Stop()) End Sub + Private Sub Checker() + Do + Try + If Count = 0 Or _StopRequested Then Exit Sub + PlansWaiter.Invoke(PlanDownloading) + Dim i% = Checker_GetNextPlanIndex() + If i >= 0 Then Checker_DownloadPlan(i) + Thread.Sleep(500) + Catch dex As ArgumentOutOfRangeException When disposedValue Or _StopRequested + Catch ex As Exception + ErrorsDescriber.Execute(EDP.SendToLog, ex, "[Scheduler.Checker]") + End Try + Loop While Not _StopRequested + _StopRequested = False + End Sub + Private Sub Checker_DownloadPlan(ByVal PlanIndex As Integer) + While Downloader.Working : Thread.Sleep(200) : End While + With Plans(PlanIndex) + If .Downloading Then + PlansWaiter.Invoke(PlanDownloading) + ElseIf .DownloadReady Then + .Download() + End If + End With + End Sub + Private Function Checker_GetNextPlanIndex() As Integer + Try + Dim result% = -1 + Dim l As New List(Of KeyValuePair(Of Integer, Date)) + Dim d As Date? + If Count > 0 Then + For i% = 0 To Count - 1 + With Plans(i) + If .DownloadReady Then + d = .NextDate + If d.HasValue Then l.Add(New KeyValuePair(Of Integer, Date)(i, d.Value)) + End If + End With + Next + End If + If l.Count > 0 Then + Dim md As Date = l.Min(Function(p) p.Value) + result = l.Find(Function(p) p.Value = md).Key + l.Clear() + End If + Return result + Catch + Return -1 + End Try + End Function Friend Property Pause(Optional ByVal LimitDate As Date? = Nothing) As PauseModes Get If Count > 0 Then Return Plans.FirstOrDefault(Function(p) p.Pause >= PauseModes.Disabled).Pause Else Return PauseModes.Disabled @@ -218,7 +281,8 @@ Namespace DownloadObjects If Not disposedValue Then If disposing Then [Stop]() - If Plans.Exists(PlanWorking) Then Task.WaitAll(Task.Run(Sub() PlansWaiter(PlanWorking))) + If Plans.Exists(PlanDownloading) Then Task.WaitAll(Task.Run(Sub() PlansWaiter(PlanDownloading))) + While Working : Thread.Sleep(200) : End While If _UpdateRequired Then Update() Plans.ListClearDispose End If diff --git a/SCrawler/Download/Automation/SchedulerEditorForm.Designer.vb b/SCrawler/Download/Automation/SchedulerEditorForm.Designer.vb index f64c789..3d92772 100644 --- a/SCrawler/Download/Automation/SchedulerEditorForm.Designer.vb +++ b/SCrawler/Download/Automation/SchedulerEditorForm.Designer.vb @@ -23,7 +23,8 @@ Namespace DownloadObjects Private Sub InitializeComponent() Dim CONTAINER_MAIN As System.Windows.Forms.ToolStripContainer - Me.LIST_PLANS = New System.Windows.Forms.ListBox() + Me.LIST_PLANS = New System.Windows.Forms.ListView() + Me.COL_MAIN = CType(New System.Windows.Forms.ColumnHeader(), System.Windows.Forms.ColumnHeader) CONTAINER_MAIN = New System.Windows.Forms.ToolStripContainer() CONTAINER_MAIN.ContentPanel.SuspendLayout() CONTAINER_MAIN.SuspendLayout() @@ -47,12 +48,25 @@ Namespace DownloadObjects ' 'LIST_PLANS ' + Me.LIST_PLANS.Alignment = System.Windows.Forms.ListViewAlignment.Left + Me.LIST_PLANS.Columns.AddRange(New System.Windows.Forms.ColumnHeader() {Me.COL_MAIN}) Me.LIST_PLANS.Dock = System.Windows.Forms.DockStyle.Fill - Me.LIST_PLANS.FormattingEnabled = True + Me.LIST_PLANS.FullRowSelect = True + Me.LIST_PLANS.HeaderStyle = System.Windows.Forms.ColumnHeaderStyle.None + Me.LIST_PLANS.HideSelection = False Me.LIST_PLANS.Location = New System.Drawing.Point(0, 0) + Me.LIST_PLANS.MultiSelect = False Me.LIST_PLANS.Name = "LIST_PLANS" + Me.LIST_PLANS.ShowGroups = False Me.LIST_PLANS.Size = New System.Drawing.Size(414, 316) Me.LIST_PLANS.TabIndex = 0 + Me.LIST_PLANS.UseCompatibleStateImageBehavior = False + Me.LIST_PLANS.View = System.Windows.Forms.View.Details + ' + 'COL_MAIN + ' + Me.COL_MAIN.Text = "Task" + Me.COL_MAIN.Width = 410 ' 'SchedulerEditorForm ' @@ -63,7 +77,6 @@ Namespace DownloadObjects Me.KeyPreview = True Me.MinimumSize = New System.Drawing.Size(430, 380) Me.Name = "SchedulerEditorForm" - Me.ShowIcon = True Me.ShowInTaskbar = False Me.Text = "Scheduler" CONTAINER_MAIN.ContentPanel.ResumeLayout(False) @@ -72,6 +85,7 @@ Namespace DownloadObjects Me.ResumeLayout(False) End Sub - Private WithEvents LIST_PLANS As ListBox + Private WithEvents LIST_PLANS As ListView + Private WithEvents COL_MAIN As ColumnHeader End Class End Namespace \ No newline at end of file diff --git a/SCrawler/Download/Automation/SchedulerEditorForm.vb b/SCrawler/Download/Automation/SchedulerEditorForm.vb index 112514c..8538a55 100644 --- a/SCrawler/Download/Automation/SchedulerEditorForm.vb +++ b/SCrawler/Download/Automation/SchedulerEditorForm.vb @@ -6,6 +6,8 @@ ' ' This program is distributed in the hope that it will be useful, ' but WITHOUT ANY WARRANTY +Imports System.Threading +Imports System.ComponentModel Imports PersonalUtilities.Forms Imports PersonalUtilities.Forms.Toolbars Imports PersonalUtilities.Forms.Controls.Base @@ -131,6 +133,16 @@ Namespace DownloadObjects SetTitle() .EndLoaderOperations(False) End With + SchedulerCheckerThread = New Thread(New ThreadStart(AddressOf SchedulerChecker)) + SchedulerCheckerThread.SetApartmentState(ApartmentState.MTA) + SchedulerCheckerThread.Start() + Try : AddHandler Settings.Automation.PlanChanged, AddressOf Scheduler_PauseChanged : Catch : End Try + End Sub + Private _CloseRequested As Boolean = False + Private Sub SchedulerEditorForm_Closing(sender As Object, e As CancelEventArgs) Handles Me.Closing + _CloseRequested = True + While If(SchedulerCheckerThread?.IsAlive, False) : Thread.Sleep(200) : End While + Try : RemoveHandler Settings.Automation.PlanChanged, AddressOf Scheduler_PauseChanged : Catch : End Try End Sub Private Sub SchedulerEditorForm_KeyDown(sender As Object, e As KeyEventArgs) Handles Me.KeyDown If e.KeyCode = Keys.Escape Then @@ -143,21 +155,48 @@ Namespace DownloadObjects Private Sub SchedulerEditorForm_Disposed(sender As Object, e As EventArgs) Handles Me.Disposed PauseArr.Dispose() End Sub + Private Sub SchedulerEditorForm_SizeChanged(sender As Object, e As EventArgs) Handles Me.SizeChanged + Try : ControlInvokeFast(LIST_PLANS, Sub() + COL_MAIN.Width = -2 + LIST_PLANS.Refresh() + End Sub, EDP.None) : Catch : End Try + End Sub #End Region + Private Property ListSelectedIndex As Integer + Get + Return If(LIST_PLANS.SelectedIndices.Count > 0, LIST_PLANS.SelectedIndices(0), -1) + End Get + Set(ByVal indx As Integer) + Try : ControlInvokeFast(LIST_PLANS, Sub() + With LIST_PLANS + .SelectedItems.Clear() + .Items(indx).Selected = True + .FocusedItem = .Items(indx) + .Refresh() + .EnsureVisible(indx) + End With + End Sub, EDP.None) : Catch : End Try + End Set + End Property Private _RefillInProgress As Boolean = False Private Sub Refill() Handles MyDefs.ButtonUpdateClick Try - If Not _RefillInProgress Then - _RefillInProgress = True - LIST_PLANS.Items.Clear() - If Settings.Automation.Count > 0 Then - LIST_PLANS.Items.AddRange(Settings.Automation.Select(Function(a) a.ToString()).Cast(Of Object).ToArray) - If _LatestSelected.ValueBetween(0, LIST_PLANS.Items.Count - 1) Then LIST_PLANS.SelectedIndex = _LatestSelected - Else - _LatestSelected = -1 + ControlInvokeFast(LIST_PLANS, + Sub() + If Not _RefillInProgress Then + _RefillInProgress = True + With LIST_PLANS.Items + .Clear() + If Settings.Automation.Count > 0 Then + .AddRange(Settings.Automation.Select(Function(a) a.LVI).ToArray) + If _LatestSelected.ValueBetween(0, .Count - 1) Then ListSelectedIndex = _LatestSelected + Else + _LatestSelected = -1 + End If + End With + _RefillInProgress = False End If - _RefillInProgress = False - End If + End Sub) Catch ex As Exception ErrorsDescriber.Execute(EDP.SendToLog, ex, "[DownloadObjects.SchedulerEditorForm.Refill]") End Try @@ -196,7 +235,7 @@ Namespace DownloadObjects Private Sub Edit() Handles MyDefs.ButtonEditClick If _LatestSelected.ValueBetween(0, LIST_PLANS.Items.Count - 1) Then Using f As New AutoDownloaderEditorForm(Settings.Automation(_LatestSelected)) : f.ShowDialog() : End Using - Refill() + LIST_PLANS_Refresh(_LatestSelected) Else MsgBoxE("You have not selected a plan to edit.", vbExclamation) End If @@ -224,14 +263,59 @@ Namespace DownloadObjects #End Region #Region "List handlers" Private _LatestSelected As Integer = -1 + Private _LatestSelectedChangeEnabled As Boolean = True Private Sub LIST_PLANS_SelectedIndexChanged(sender As Object, e As EventArgs) Handles LIST_PLANS.SelectedIndexChanged - _LatestSelected = LIST_PLANS.SelectedIndex - PauseArr.PlanIndex = _LatestSelected - PauseArr.UpdatePauseButtons(False) + If _LatestSelectedChangeEnabled Then + _LatestSelected = ListSelectedIndex + PauseArr.PlanIndex = _LatestSelected + PauseArr.UpdatePauseButtons(False) + End If End Sub Private Sub LIST_PLANS_MouseDoubleClick(sender As Object, e As MouseEventArgs) Handles LIST_PLANS.MouseDoubleClick Edit() End Sub + Private Sub LIST_PLANS_Refresh(Optional ByVal PlanIndex As Integer = -1) + _LatestSelectedChangeEnabled = False + Try + ControlInvokeFast(LIST_PLANS, + Sub() + Dim indx% = If(PlanIndex >= 0, PlanIndex, ListSelectedIndex) + If indx = -1 And PlanIndex = -1 Then indx = _LatestSelected + If indx.ValueBetween(0, Settings.Automation.Count - 1) Then _ + LIST_PLANS.Items(indx) = Settings.Automation(indx).LVI + If _LatestSelected.ValueBetween(0, LIST_PLANS.Items.Count - 1) Then ListSelectedIndex = _LatestSelected + End Sub, EDP.None) + Catch + Finally + _LatestSelectedChangeEnabled = True + End Try + End Sub + Private Sub Scheduler_PauseChanged(ByVal Plan As AutoDownloader) + LIST_PLANS_Refresh(Plan.Index) + End Sub + Private SchedulerCheckerThread As Thread + Private Sub SchedulerChecker() + Try + While Not _CloseRequested + Try + With Settings.Automation + If .Count > 0 Then + For i% = 0 To .Count - 1 + If .Item(i).LVIStateChanged Then LIST_PLANS_Refresh(i) + If _CloseRequested Then Exit Sub + Thread.Sleep(200) + Next + Else + If _CloseRequested Then Exit Sub + Thread.Sleep(200) + End If + End With + Catch + End Try + End While + Catch + End Try + End Sub #End Region #Region "Settings, Start, Skip, Pause" Private Function GetSchedulerFiles() As List(Of SFile) @@ -372,13 +456,13 @@ Namespace DownloadObjects Private Sub BTT_START_Click(sender As Object, e As EventArgs) Handles BTT_START.Click If _LatestSelected.ValueBetween(0, LIST_PLANS.Items.Count - 1) Then With Settings.Automation(_LatestSelected) : .Start(.IsNewPlan) : End With - Refill() + LIST_PLANS_Refresh(_LatestSelected) End If End Sub Private Sub BTT_START_FORCE_Click(sender As Object, e As EventArgs) Handles BTT_START_FORCE.Click If _LatestSelected.ValueBetween(0, LIST_PLANS.Items.Count - 1) Then With Settings.Automation(_LatestSelected) - If .Working Or .IsManual Then .ForceStart() : Refill() + If .Working Or .IsManual Then .ForceStart() : LIST_PLANS_Refresh(_LatestSelected) End With End If End Sub @@ -388,10 +472,10 @@ Namespace DownloadObjects Select Case mode Case String.Empty Settings.Automation(_LatestSelected).Skip() - Refill() + LIST_PLANS_Refresh(_LatestSelected) Case "m" Dim mins% = AConvert(Of Integer)(InputBoxE("Enter a number of minutes you want to delay:", Sender.Text, 60), -1) - If mins > 0 Then Settings.Automation(_LatestSelected).Skip(mins) : Refill() + If mins > 0 Then Settings.Automation(_LatestSelected).Skip(mins) : LIST_PLANS_Refresh(_LatestSelected) Case "d" Dim d As Date? = Nothing Using f As New DateTimeSelectionForm(DateTimeSelectionForm.Modes.Date + @@ -402,16 +486,13 @@ Namespace DownloadObjects f.ShowDialog() If f.DialogResult = DialogResult.OK Then d = f.MyDateStart End Using - If d.HasValue Then Settings.Automation(_LatestSelected).Skip(d.Value) : Refill() + If d.HasValue Then Settings.Automation(_LatestSelected).Skip(d.Value) : LIST_PLANS_Refresh(_LatestSelected) Case "r" Settings.Automation(_LatestSelected).SkipReset() - Refill() + LIST_PLANS_Refresh(_LatestSelected) End Select End If End Sub - Private Sub PauseArr_Updating() Handles PauseArr.Updating - Refill() - End Sub #End Region #Region "Move" Private Sub BTT_MOVE_UP_DOWN_Click(sender As Object, e As EventArgs) Handles BTT_MOVE_UP.Click, BTT_MOVE_DOWN.Click diff --git a/SCrawler/Download/Feed/DownloadFeedForm.vb b/SCrawler/Download/Feed/DownloadFeedForm.vb index 7fc2ad1..8b7b439 100644 --- a/SCrawler/Download/Feed/DownloadFeedForm.vb +++ b/SCrawler/Download/Feed/DownloadFeedForm.vb @@ -276,7 +276,6 @@ Namespace DownloadObjects End With Catch ex As Exception ErrorsDescriber.Execute(EDP.SendToLog, ex, "Feed removed") - MainFrameObj.UpdateLogButton() End Try End Sub Private Sub Feed_SPEC_LOAD(ByVal Source As ToolStripMenuItem, ByVal e As EventArgs) @@ -861,7 +860,6 @@ Namespace DownloadObjects End If Catch ex As Exception ErrorsDescriber.Execute(EDP.SendToLog, ex, "Delete daily sessions (by list)") - MainFrameObj.UpdateLogButton() End Try End Sub Private Sub BTT_FEED_DELETE_DAILY_DATE_Click(sender As Object, e As EventArgs) Handles BTT_FEED_DELETE_DAILY_DATE.Click @@ -899,7 +897,6 @@ Namespace DownloadObjects End Using Catch ex As Exception ErrorsDescriber.Execute(EDP.SendToLog, ex, "Delete daily sessions (by date)") - MainFrameObj.UpdateLogButton() End Try End Sub #End Region diff --git a/SCrawler/Download/Feed/FeedSpecial.vb b/SCrawler/Download/Feed/FeedSpecial.vb index 3bf57ec..ae8d704 100644 --- a/SCrawler/Download/Feed/FeedSpecial.vb +++ b/SCrawler/Download/Feed/FeedSpecial.vb @@ -139,7 +139,6 @@ Namespace DownloadObjects End If Catch ex As Exception ErrorsDescriber.Execute(EDP.SendToLog, ex, "[FeedSpecial.UpdateUsers]") - MainFrameObj.UpdateLogButton() End Try End Sub Friend Overloads Shared Function UpdateUsers(ByVal Item As UserMediaD, ByVal InitialUser As UserInfo, ByVal NewUser As UserInfo, diff --git a/SCrawler/Download/Feed/FeedSpecialCollection.vb b/SCrawler/Download/Feed/FeedSpecialCollection.vb index 417ba28..38fb7e3 100644 --- a/SCrawler/Download/Feed/FeedSpecialCollection.vb +++ b/SCrawler/Download/Feed/FeedSpecialCollection.vb @@ -111,7 +111,6 @@ Namespace DownloadObjects End If Catch ex As Exception ErrorsDescriber.Execute(EDP.SendToLog, ex, "[FeedSpecialCollection.Load]") - MainFrameObj.UpdateLogButton() End Try End Sub Private Sub Feeds_FeedDeleted(ByVal Source As FeedSpecialCollection, ByVal Feed As FeedSpecial) @@ -245,7 +244,6 @@ Namespace DownloadObjects End If Catch ex As Exception ErrorsDescriber.Execute(EDP.SendToLog, ex, "[FeedSpecialCollection.UpdateUsers]") - MainFrameObj.UpdateLogButton() End Try End Sub #End Region diff --git a/SCrawler/Download/STDownloader/DownloaderUrlForm.vb b/SCrawler/Download/STDownloader/DownloaderUrlForm.vb index 54fcbd6..8241511 100644 --- a/SCrawler/Download/STDownloader/DownloaderUrlForm.vb +++ b/SCrawler/Download/STDownloader/DownloaderUrlForm.vb @@ -93,7 +93,6 @@ Namespace DownloadObjects.STDownloader End With Catch ex As Exception ErrorsDescriber.Execute(EDP.SendToLog, ex, "[STDownloader.DownloaderUrlForm.TXT_URL_ActionOnTextChanged]") - MainFrameObj.UpdateLogButton() End Try End Sub Private Sub TXT_PATH_ActionOnButtonClick(ByVal Sender As ActionButton, ByVal e As ActionButtonEventArgs) Handles TXT_PATH.ActionOnButtonClick diff --git a/SCrawler/Download/STDownloader/DownloaderUrlsArrForm.vb b/SCrawler/Download/STDownloader/DownloaderUrlsArrForm.vb index 389d414..3d11b08 100644 --- a/SCrawler/Download/STDownloader/DownloaderUrlsArrForm.vb +++ b/SCrawler/Download/STDownloader/DownloaderUrlsArrForm.vb @@ -108,7 +108,6 @@ Namespace DownloadObjects.STDownloader Catch ex As Exception _UseAccountName = True ErrorsDescriber.Execute(EDP.SendToLog, ex, "[STDownloader.DownloaderUrlsArrForm.TXT_URLS_TextChanged]") - MainFrameObj.UpdateLogButton() End Try End Sub End Class diff --git a/SCrawler/Download/STDownloader/VideoDownloaderForm.vb b/SCrawler/Download/STDownloader/VideoDownloaderForm.vb index 509ea50..ff18c12 100644 --- a/SCrawler/Download/STDownloader/VideoDownloaderForm.vb +++ b/SCrawler/Download/STDownloader/VideoDownloaderForm.vb @@ -226,9 +226,5 @@ Namespace DownloadObjects.STDownloader RemoveControls(ControlNonYT, False) End If End Sub - Protected Overrides Sub MyJob_Finished(ByVal Sender As Object, ByVal e As EventArgs) - MainFrameObj.UpdateLogButton() - MyBase.MyJob_Finished(Sender, e) - End Sub End Class End Namespace \ No newline at end of file diff --git a/SCrawler/Download/TDownloader.vb b/SCrawler/Download/TDownloader.vb index d3aa598..f3233ed 100644 --- a/SCrawler/Download/TDownloader.vb +++ b/SCrawler/Download/TDownloader.vb @@ -228,7 +228,6 @@ Namespace DownloadObjects Catch ex As Exception _FilesUpdating = False ErrorsDescriber.Execute(EDP.SendToLog, ex, "[TDownloader.FilesUpdatePendingUsers]") - MainFrameObj.UpdateLogButton() End Try End Sub Friend Sub ClearSessions() @@ -536,10 +535,8 @@ Namespace DownloadObjects If Pool.Count > 0 Then Pool.ForEach(Sub(p) If Not p.Progress Is Nothing Then p.Progress.Maximum = 0) ExecuteCommand(Settings.DownloadsCompleteCommand) UpdateJobsLabel() - If MissingPostsDetected And Settings.AddMissingToLog Then - MyMainLOG = "Some posts didn't download. You can see them in the 'Missing posts' form." - MainFrameObj.UpdateLogButton() - End If + If MissingPostsDetected And Settings.AddMissingToLog Then _ + MyMainLOG = "Some posts didn't download. You can see them in the 'Missing posts' form." Files.Sort() FilesChanged = Not fBefore = Files.Count RaiseEvent Downloading(False) @@ -612,6 +609,7 @@ Namespace DownloadObjects Dim limitIndex% Dim limits As New List(Of HostLimit) Dim Keys As New List(Of String) + Dim KeysSkipped As New List(Of String) Dim h As Boolean = False Dim host As SettingsHost = Nothing Dim hostAvailable As Boolean @@ -638,6 +636,8 @@ Namespace DownloadObjects limit = limit.Next limits(limitIndex) = limit If limit.Value >= limit.Limit Then Exit For + Else + KeysSkipped.Add(_Item.Key) End If End If End If @@ -655,24 +655,27 @@ Namespace DownloadObjects For Each k$ In Keys i = _Job.Items.FindIndex(Function(ii) ii.Key = k) If i >= 0 Then - With _Job.Items(i) - If DirectCast(.Self, UserDataBase).ContentMissingExists Then MissingPostsDetected = True - RaiseEvent UserDownloadStateChanged(.Self, False) - host = _Job.UserHost(.Self) - host.AfterDownload(.Self, Download.Main) - If Not .Disposed AndAlso Not .IsCollection AndAlso .DownloadedTotal(False) > 0 Then - If Not Downloaded.Contains(.Self) Then Downloaded.Add(Settings.GetUser(.Self)) - With DirectCast(.Self, UserDataBase) - If .LatestData.Count > 0 And .IncludeInTheFeed Then Files.ListAddList(.LatestData.Select(Function(d) New UserMediaD(d, .Self, Session)), FilesLP) - End With - dcc = True - End If - End With + If KeysSkipped.Count = 0 OrElse Not KeysSkipped.Contains(k) Then + With _Job.Items(i) + If DirectCast(.Self, UserDataBase).ContentMissingExists Then MissingPostsDetected = True + RaiseEvent UserDownloadStateChanged(.Self, False) + host = _Job.UserHost(.Self) + host.AfterDownload(.Self, Download.Main) + If Not .Disposed AndAlso Not .IsCollection AndAlso .DownloadedTotal(False) > 0 Then + If Not Downloaded.Contains(.Self) Then Downloaded.Add(Settings.GetUser(.Self)) + With DirectCast(.Self, UserDataBase) + If .LatestData.Count > 0 And .IncludeInTheFeed Then Files.ListAddList(.LatestData.Select(Function(d) New UserMediaD(d, .Self, Session)), FilesLP) + End With + dcc = True + End If + End With + End If _Job.Items.RemoveAt(i) End If Next End If Keys.Clear() + KeysSkipped.Clear() _Job.Items.RemoveAll(Function(ii) ii.Disposed) If dcc Then Downloaded.RemoveAll(Function(u) u Is Nothing) If dcc And Downloaded.Count > 0 Then RaiseEvent DownloadCountChange() diff --git a/SCrawler/Editors/GlobalSettingsForm.Designer.vb b/SCrawler/Editors/GlobalSettingsForm.Designer.vb index 031d387..087de6a 100644 --- a/SCrawler/Editors/GlobalSettingsForm.Designer.vb +++ b/SCrawler/Editors/GlobalSettingsForm.Designer.vb @@ -167,6 +167,7 @@ Namespace Editors Me.TXT_DOWN_COMPLETE_SCRIPT = New PersonalUtilities.Forms.Controls.TextBoxExtended() Me.CH_UNAME_UP = New System.Windows.Forms.CheckBox() Me.CH_UICON_UP = New System.Windows.Forms.CheckBox() + Me.TXT_AUTO_BRUSH_MIN = New PersonalUtilities.Forms.Controls.TextBoxExtended() Me.TXT_FEED_ROWS = New PersonalUtilities.Forms.Controls.TextBoxExtended() Me.TXT_FEED_COLUMNS = New PersonalUtilities.Forms.Controls.TextBoxExtended() Me.CH_FEED_ENDLESS = New System.Windows.Forms.CheckBox() @@ -269,6 +270,7 @@ Namespace Editors CType(Me.TXT_SCRIPT, System.ComponentModel.ISupportInitialize).BeginInit() CType(Me.TXT_DOWN_COMPLETE_SCRIPT, System.ComponentModel.ISupportInitialize).BeginInit() TP_MISSING_DATA.SuspendLayout() + CType(Me.TXT_AUTO_BRUSH_MIN, System.ComponentModel.ISupportInitialize).BeginInit() TAB_FEED.SuspendLayout() TP_FEED.SuspendLayout() TP_FEED_IMG_COUNT.SuspendLayout() @@ -1488,10 +1490,11 @@ Namespace Editors TP_DOWNLOADING.Controls.Add(Me.CH_UNAME_UP, 0, 1) TP_DOWNLOADING.Controls.Add(Me.CH_UICON_UP, 0, 2) TP_DOWNLOADING.Controls.Add(Me.CH_USE_DEF_ACC, 0, 9) + TP_DOWNLOADING.Controls.Add(Me.TXT_AUTO_BRUSH_MIN, 0, 10) TP_DOWNLOADING.Dock = System.Windows.Forms.DockStyle.Fill TP_DOWNLOADING.Location = New System.Drawing.Point(0, 0) TP_DOWNLOADING.Name = "TP_DOWNLOADING" - TP_DOWNLOADING.RowCount = 11 + TP_DOWNLOADING.RowCount = 12 TP_DOWNLOADING.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!)) TP_DOWNLOADING.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!)) TP_DOWNLOADING.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!)) @@ -1502,6 +1505,7 @@ Namespace Editors TP_DOWNLOADING.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!)) TP_DOWNLOADING.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!)) TP_DOWNLOADING.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!)) + TP_DOWNLOADING.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!)) TP_DOWNLOADING.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!)) TP_DOWNLOADING.Size = New System.Drawing.Size(621, 399) TP_DOWNLOADING.TabIndex = 1 @@ -1584,6 +1588,24 @@ Namespace Editors Me.CH_UICON_UP.Text = "Update user icon and banner every time (where supported)" Me.CH_UICON_UP.UseVisualStyleBackColor = True ' + 'TXT_AUTO_BRUSH_MIN + ' + Me.TXT_AUTO_BRUSH_MIN.CaptionText = "Highlight undownloaded plans (minutes)" + Me.TXT_AUTO_BRUSH_MIN.CaptionToolTipEnabled = True + Me.TXT_AUTO_BRUSH_MIN.CaptionToolTipText = "Highlight (in gray) the scheduler plans that have not been downloaded in 'x' minu" & + "tes." & Global.Microsoft.VisualBasic.ChrW(13) & Global.Microsoft.VisualBasic.ChrW(10) & "-1 to disable." + Me.TXT_AUTO_BRUSH_MIN.CaptionWidth = 200.0R + Me.TXT_AUTO_BRUSH_MIN.ControlMode = PersonalUtilities.Forms.Controls.TextBoxExtended.ControlModes.NumericUpDown + Me.TXT_AUTO_BRUSH_MIN.Dock = System.Windows.Forms.DockStyle.Fill + Me.TXT_AUTO_BRUSH_MIN.Location = New System.Drawing.Point(4, 280) + Me.TXT_AUTO_BRUSH_MIN.Name = "TXT_AUTO_BRUSH_MIN" + Me.TXT_AUTO_BRUSH_MIN.NumberMaximum = New Decimal(New Integer() {2147483646, 0, 0, 0}) + Me.TXT_AUTO_BRUSH_MIN.NumberMinimum = New Decimal(New Integer() {1, 0, 0, -2147483648}) + Me.TXT_AUTO_BRUSH_MIN.NumberUpDownAlign = System.Windows.Forms.LeftRightAlignment.Left + Me.TXT_AUTO_BRUSH_MIN.Size = New System.Drawing.Size(613, 22) + Me.TXT_AUTO_BRUSH_MIN.TabIndex = 10 + Me.TXT_AUTO_BRUSH_MIN.Text = "-1" + ' 'TAB_FEED ' TAB_FEED.Controls.Add(TP_FEED) @@ -2486,6 +2508,7 @@ Namespace Editors CType(Me.TXT_DOWN_COMPLETE_SCRIPT, System.ComponentModel.ISupportInitialize).EndInit() TP_MISSING_DATA.ResumeLayout(False) TP_MISSING_DATA.PerformLayout() + CType(Me.TXT_AUTO_BRUSH_MIN, System.ComponentModel.ISupportInitialize).EndInit() TAB_FEED.ResumeLayout(False) TP_FEED.ResumeLayout(False) TP_FEED.PerformLayout() @@ -2636,5 +2659,6 @@ Namespace Editors Private WithEvents CH_DOWN_ALL_NOTIFY As CheckBox Private WithEvents CH_CHANNELS_USERS_READY_MARK As CheckBox Private WithEvents NUM_FEED_SES_CURR_LOAD_LAST As PersonalUtilities.Forms.Controls.TextBoxExtended + Private WithEvents TXT_AUTO_BRUSH_MIN As PersonalUtilities.Forms.Controls.TextBoxExtended End Class End Namespace \ No newline at end of file diff --git a/SCrawler/Editors/GlobalSettingsForm.vb b/SCrawler/Editors/GlobalSettingsForm.vb index 2ca3930..6dd713b 100644 --- a/SCrawler/Editors/GlobalSettingsForm.vb +++ b/SCrawler/Editors/GlobalSettingsForm.vb @@ -123,6 +123,7 @@ Namespace Editors CH_ADD_MISSING_ERROS_TO_LOG.Checked = .AddMissingErrorsToLog CH_DOWN_REPARSE_MISSING.Checked = .ReparseMissingInTheRoutine CH_USE_DEF_ACC.Checked = .UseDefaultAccountIfMissing + TXT_AUTO_BRUSH_MIN.Value = .AutomationBrushUndownloadedPlansMinutes.Value 'Downloading: file names CH_FILE_NAME_CHANGE.Checked = Not .FileReplaceNameByDate.Value = FileNameReplaceMode.None OPT_FILE_NAME_REPLACE.Checked = .FileReplaceNameByDate.Value = FileNameReplaceMode.Replace @@ -331,6 +332,7 @@ Namespace Editors .AddMissingErrorsToLog.Value = CH_ADD_MISSING_ERROS_TO_LOG.Checked .ReparseMissingInTheRoutine.Value = CH_DOWN_REPARSE_MISSING.Checked .UseDefaultAccountIfMissing.Value = CH_USE_DEF_ACC.Checked + .AutomationBrushUndownloadedPlansMinutes.Value = TXT_AUTO_BRUSH_MIN.Value 'Downloading: file names If CH_FILE_NAME_CHANGE.Checked Then .FileReplaceNameByDate.Value = If(OPT_FILE_NAME_REPLACE.Checked, FileNameReplaceMode.Replace, FileNameReplaceMode.Add) diff --git a/SCrawler/Editors/UsersInfoForm.vb b/SCrawler/Editors/UsersInfoForm.vb index 4632a80..fdc7e29 100644 --- a/SCrawler/Editors/UsersInfoForm.vb +++ b/SCrawler/Editors/UsersInfoForm.vb @@ -313,7 +313,6 @@ Namespace Editors BTT_START.Enabled = Not Working BTT_CANCEL.Enabled = Working End Sub, EDP.None) - If Not Working Then MainFrameObj.UpdateLogButton() Catch End Try End Sub diff --git a/SCrawler/MainFrame.vb b/SCrawler/MainFrame.vb index b3c3241..02f0e83 100644 --- a/SCrawler/MainFrame.vb +++ b/SCrawler/MainFrame.vb @@ -131,7 +131,6 @@ Public Class MainFrame Await .Automation.Start(True) End With UpdatePauseButtonsVisibility() - MainFrameObj.UpdateLogButton() End Sub Private _CloseInvoked As Boolean = False Private _IgnoreTrayOptions As Boolean = False @@ -625,7 +624,6 @@ CloseResume: End Using Catch ex As Exception ErrorsDescriber.Execute(EDP.SendToLog, ex, "[MainFrame.ShowGroups]") - MainFrameObj.UpdateLogButton() End Try End Sub #End Region @@ -652,7 +650,6 @@ CloseResume: MainFrameObj.PauseButtons.UpdatePauseButtons() Catch ex As Exception ErrorsDescriber.Execute(EDP.LogMessageValue, ex, "Start automation") - MainFrameObj.UpdateLogButton() End Try End Sub Private Sub BTT_DOWN_AUTOMATION_PAUSE_Click(sender As Object, e As EventArgs) Handles BTT_DOWN_AUTOMATION_PAUSE.Click, BTT_TRAY_PAUSE_AUTOMATION.Click @@ -2022,7 +2019,6 @@ ResumeDownloadingOperation: End Sub Private Sub Downloader_UpdateJobsCount(ByVal TotalCount As Integer) ControlInvokeFast(Toolbar_BOTTOM, LBL_JOBS_COUNT, Sub() LBL_JOBS_COUNT.Text = IIf(TotalCount = 0, String.Empty, $"[Jobs {TotalCount}]")) - MainFrameObj.UpdateLogButton() End Sub Private Sub Downloader_Downloading(ByVal Value As Boolean) Dim __isDownloading As Boolean = Value Or Downloader.Working(False) diff --git a/SCrawler/MainFrameObjects.vb b/SCrawler/MainFrameObjects.vb index 618f82c..b46ff38 100644 --- a/SCrawler/MainFrameObjects.vb +++ b/SCrawler/MainFrameObjects.vb @@ -66,17 +66,17 @@ Friend Class MainFrameObjects : Implements INotificator ControlInvokeFast(MF.TRAY_CONTEXT, Sub() MF.BTT_TRAY_CLOSE_NO_SCRIPT.Visible = Settings.ClosingCommand.Attribute And Not Settings.ClosingCommand.IsEmptyString) End Sub - Friend Sub UpdateLogButton() - MyMainLOG_UpdateLogButton(MF.BTT_LOG, MF.Toolbar_TOP) + Private Sub UpdateLogButton() + Try : MyMainLOG_UpdateLogButton(MF.BTT_LOG, MF.Toolbar_TOP) : Catch : End Try End Sub Friend Function GetUserListProvider(ByVal WithCollections As Boolean) As IFormatProvider Return MF.GetUserListProvider(WithCollections) End Function Friend Sub ShowLog() - MyMainLOG_ShowForm(Settings.Design,,,, Sub() - UpdateLogButton() - LogFormClosed() - End Sub) + Try : MyMainLOG_ShowForm(Settings.Design,,,, Sub() + UpdateLogButton() + LogFormClosed() + End Sub) : Catch : End Try End Sub #End Region #Region "Notifications" diff --git a/SCrawler/PluginsEnvironment/Hosts/SettingsHostCollection.vb b/SCrawler/PluginsEnvironment/Hosts/SettingsHostCollection.vb index a485b1f..9a280c0 100644 --- a/SCrawler/PluginsEnvironment/Hosts/SettingsHostCollection.vb +++ b/SCrawler/PluginsEnvironment/Hosts/SettingsHostCollection.vb @@ -202,7 +202,6 @@ Namespace Plugin.Hosts Select Case Hosts_Deleted_MoveAcc(Obj) Case -1 : ShowOperationCanceledMsg(ChngUACC_MsgTitle) : Exit Sub Case 1 - MainFrameObj.UpdateLogButton() MsgBoxE({$"An error occurred while changing user accounts (see log for details).{vbCr}Operation canceled.", ChngUACC_MsgTitle}, vbCritical) Exit Sub End Select diff --git a/SCrawler/SettingsCLS.vb b/SCrawler/SettingsCLS.vb index f67a475..33db66e 100644 --- a/SCrawler/SettingsCLS.vb +++ b/SCrawler/SettingsCLS.vb @@ -356,6 +356,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable DownloadsCompleteCommand = New XMLValueAttribute(Of String, Boolean)("DownloadsCompleteCommand", "Use",,, MyXML, n) ReparseMissingInTheRoutine = New XMLValue(Of Boolean)("ReparseMissingInTheRoutine", False, MyXML, n) UseDefaultAccountIfMissing = New XMLValue(Of Boolean)("UseDefaultAccountIfMissing", True, MyXML, n) + AutomationBrushUndownloadedPlansMinutes = New XMLValue(Of Integer)("AutomationBrushUndownloadedPlansMinutes", 10080, MyXML, n) 'Downloading: file naming n_old = {"Users", "FileName"} @@ -1001,6 +1002,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable End Get End Property #End Region +#Region "Properties" #Region "Basis" Friend ReadOnly Property GlobalPath As XMLValue(Of SFile) Friend ReadOnly Property MaxLargeImageHeight As XMLValue(Of Integer) @@ -1204,6 +1206,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable Friend ReadOnly Property DownloadsCompleteCommand As XMLValueAttribute(Of String, Boolean) Friend ReadOnly Property ReparseMissingInTheRoutine As XMLValue(Of Boolean) Friend ReadOnly Property UseDefaultAccountIfMissing As XMLValue(Of Boolean) + Friend ReadOnly Property AutomationBrushUndownloadedPlansMinutes As XMLValue(Of Integer) #End Region #Region "Downloading: file naming" Friend ReadOnly Property FileAddDateToFileName As XMLValue(Of Boolean) @@ -1309,6 +1312,7 @@ Friend Class SettingsCLS : Implements IDownloaderSettings, IDisposable Friend ReadOnly Property LatestSavingPath As XMLValue(Of SFile) Friend ReadOnly Property LatestSelectedChannel As XMLValue(Of String) #End Region +#End Region #Region "IDisposable Support" Private disposedValue As Boolean = False Protected Overridable Overloads Sub Dispose(ByVal disposing As Boolean)