geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-14 15:52:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

2023.11.21.0

Browse Source 
YouTube
Add absolute paths support
PlaylistArrayForm: fix RTB issue
Single media: the file name is not changed manually

SCrawler
Automation: add manual tasks
DownloaderUrlsArrForm: fix RTB issue
SiteSettingsBase: add use of Netscape cookies if enabled for a class; disable saving Netscape cookies on init
Add feeds update when users' location and/or basic info changes
API.TikTok: add ID, username and friendly name extraction from data; update request URL; update 'GetUserUrl' function; add new option 'Use video date as file date'
API.YouTube: set 'UseNetscapeCookies'
This commit is contained in:
Andy
2023-11-21 08:38:37 +03:00
parent 45adf735a7
commit 0ec617c1dc
26 changed files with 359 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
SCrawler.PluginProvider/.editorconfig
View File

@@ -1,3 +1,3 @@
[*.vb]
# Modifier preferences
file_header_template = Copyright (C) 2023 Andy https://github.com/AAndyProgram\nThis program is free software: you can redistribute it and/or modify\nit under the terms of the GNU General Public License as published by\nthe Free Software Foundation, either version 3 of the License, or\n(at your option) any later version.\n\nThis program is distributed in the hope that it will be useful,\nbut WITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\nGNU General Public License for more details.\n\nYou should have received a copy of the GNU General Public License\nalong with this program. If not, see <https://www.gnu.org/licenses/>
file_header_template = Copyright (C) Andy https://github.com/AAndyProgram\nThis program is free software: you can redistribute it and/or modify\nit under the terms of the GNU General Public License as published by\nthe Free Software Foundation, either version 3 of the License, or\n(at your option) any later version.\n\nThis program is distributed in the hope that it will be useful,\nbut WITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\nGNU General Public License for more details.\n\nYou should have received a copy of the GNU General Public License\nalong with this program. If not, see <https://www.gnu.org/licenses/>

2
SCrawler.YouTube/.editorconfig
View File

@@ -1,3 +1,3 @@
[*.vb]
# Modifier preferences
file_header_template = Copyright (C) 2023 Andy https://github.com/AAndyProgram\nThis program is free software: you can redistribute it and/or modify\nit under the terms of the GNU General Public License as published by\nthe Free Software Foundation, either version 3 of the License, or\n(at your option) any later version.\n\nThis program is distributed in the hope that it will be useful,\nbut WITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\nGNU General Public License for more details.\n\nYou should have received a copy of the GNU General Public License\nalong with this program. If not, see <https://www.gnu.org/licenses/>
file_header_template = Copyright (C) Andy https://github.com/AAndyProgram\nThis program is free software: you can redistribute it and/or modify\nit under the terms of the GNU General Public License as published by\nthe Free Software Foundation, either version 3 of the License, or\n(at your option) any later version.\n\nThis program is distributed in the hope that it will be useful,\nbut WITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\nGNU General Public License for more details.\n\nYou should have received a copy of the GNU General Public License\nalong with this program. If not, see <https://www.gnu.org/licenses/>

6
SCrawler.YouTube/Controls/MusicPlaylistsForm.Designer.vb generated
View File

@@ -427,10 +427,14 @@ Namespace API.YouTube.Controls
Me.TXT_OUTPUT_PATH.Buttons.Add(ActionButton8)
Me.TXT_OUTPUT_PATH.Buttons.Add(ActionButton9)
Me.TXT_OUTPUT_PATH.Buttons.Add(ActionButton10)
Me.TXT_OUTPUT_PATH.CaptionMode = PersonalUtilities.Forms.Controls.Base.ICaptionControl.Modes.Label
Me.TXT_OUTPUT_PATH.CaptionMode = PersonalUtilities.Forms.Controls.Base.ICaptionControl.Modes.CheckBox
Me.TXT_OUTPUT_PATH.CaptionText = "Output path"
Me.TXT_OUTPUT_PATH.CaptionToolTipEnabled = True
Me.TXT_OUTPUT_PATH.CaptionToolTipText = "If this checkbox is selected, this path is absolute and artist folder will not be" &
" created in it"
Me.TXT_OUTPUT_PATH.CaptionVisible = True
Me.TXT_OUTPUT_PATH.CaptionWidth = 112.0R
Me.TXT_OUTPUT_PATH.ChangeControlsEnableOnCheckedChange = False
ListColumn1.Name = "COL_NAME"
ListColumn1.Text = "Name"
ListColumn1.Width = -1

9
SCrawler.YouTube/Controls/MusicPlaylistsForm.vb
View File

@@ -80,6 +80,14 @@ Namespace API.YouTube.Controls
End If
LIST_PLAYLISTS.SelectedIndex = 0
If .ObjectType = Base.YouTubeMediaType.Channel Then
With TXT_OUTPUT_PATH
.CaptionMode = ICaptionControl.Modes.Label
.CaptionToolTipText = String.Empty
.CaptionToolTipEnabled = False
End With
End If
TXT_OUTPUT_PATH.Text = MyYouTubeSettings.OutputPath.Value
If Not .UserTitle.IsEmptyString Then
@@ -266,6 +274,7 @@ Namespace API.YouTube.Controls
If Not TXT_SUBS.Checked Then .PostProcessing_OutputSubtitlesFormats.Clear()
.OutputAudioCodec = CMB_FORMATS.Text
If Not TXT_FORMATS_ADDIT.Checked Then .PostProcessing_OutputAudioFormats.Clear()
.AbsolutePath = TXT_OUTPUT_PATH.Checked
.File = TXT_OUTPUT_PATH.Text.CSFileP
If MyYouTubeSettings.OutputPathAutoChange Then MyYouTubeSettings.OutputPath.Value = .File
If MyDownloaderSettings.OutputPathAutoAddPaths Then MyYouTubeSettings.DownloadLocations.Add(.File, False)

10
SCrawler.YouTube/Controls/PlaylistArrayForm.Designer.vb generated
View File

@@ -26,7 +26,7 @@ Namespace API.YouTube.Controls
Dim TP_MAIN As System.Windows.Forms.TableLayoutPanel
Dim FRM_PLS As System.Windows.Forms.GroupBox
Me.CH_PLS_ONE = New System.Windows.Forms.CheckBox()
Me.TXT_URLS = New System.Windows.Forms.RichTextBox()
Me.TXT_URLS = New System.Windows.Forms.TextBox()
CONTAINER_MAIN = New System.Windows.Forms.ToolStripContainer()
TP_MAIN = New System.Windows.Forms.TableLayoutPanel()
FRM_PLS = New System.Windows.Forms.GroupBox()
@@ -94,13 +94,14 @@ Namespace API.YouTube.Controls
'
'TXT_URLS
'
Me.TXT_URLS.DetectUrls = False
Me.TXT_URLS.AcceptsReturn = True
Me.TXT_URLS.Dock = System.Windows.Forms.DockStyle.Fill
Me.TXT_URLS.Location = New System.Drawing.Point(3, 16)
Me.TXT_URLS.MaxLength = 2147483647
Me.TXT_URLS.Multiline = True
Me.TXT_URLS.Name = "TXT_URLS"
Me.TXT_URLS.Size = New System.Drawing.Size(372, 261)
Me.TXT_URLS.TabIndex = 0
Me.TXT_URLS.Text = ""
'
'PlaylistArrayForm
'
@@ -119,10 +120,11 @@ Namespace API.YouTube.Controls
TP_MAIN.ResumeLayout(False)
TP_MAIN.PerformLayout()
FRM_PLS.ResumeLayout(False)
FRM_PLS.PerformLayout()
Me.ResumeLayout(False)
End Sub
Private WithEvents CH_PLS_ONE As CheckBox
Private WithEvents TXT_URLS As RichTextBox
Private WithEvents TXT_URLS As TextBox
End Class
End Namespace

1
SCrawler.YouTube/Controls/VideoOptionsForm.Designer.vb generated
View File

@@ -275,6 +275,7 @@ Namespace API.YouTube.Controls
ActionButton1.Name = "ArrowDown"
ActionButton1.Tag = PersonalUtilities.Forms.Controls.Base.ActionButton.DefaultButtons.ArrowDown
Me.TXT_FILE.Buttons.Add(ActionButton1)
Me.TXT_FILE.ChangeControlsEnableOnCheckedChange = False
ListColumn1.Name = "COL_NAME"
ListColumn1.Text = "Name"
ListColumn1.Width = -1

9
SCrawler.YouTube/Controls/VideoOptionsForm.vb
View File

@@ -76,6 +76,12 @@ Namespace API.YouTube.Controls
If Not def.ValueBetween(-1, 10000) Then def = 1080
End If
NUM_RES.Value = def
With TXT_FILE
.CaptionMode = ICaptionControl.Modes.CheckBox
.CaptionWidth = 18
.CaptionToolTipText = "If this checkbox is selected, this path is absolute and artist folder will not be created in it"
.CaptionToolTipEnabled = True
End With
Else
TP_OPTIONS.Controls.Remove(NUM_RES)
TP_OPTIONS.ColumnStyles(3).Width = 0
@@ -297,8 +303,8 @@ Namespace API.YouTube.Controls
.SelectedVideoIndex = -1
.SelectedAudioIndex = cntIndex
End If
.File = f
.FileSetManually = True
.File = f
.UpdateInfoFields()
'#If DEBUG Then
'Debug.WriteLine(.Command(False))
@@ -309,6 +315,7 @@ Namespace API.YouTube.Controls
Else
.SetMaxResolution(NUM_RES.Value)
End If
.AbsolutePath = TXT_FILE.Checked
.File = f
End If
End With

19
SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb
View File

@@ -543,6 +543,16 @@ Namespace API.YouTube.Objects
Return _FileIsPlaylistObject
End Get
End Property
Private _AbsolutePath As Boolean = False
Public Property AbsolutePath As Boolean
Get
Return _AbsolutePath
End Get
Set(ByVal ap As Boolean)
_AbsolutePath = ap
If Elements.Count > 0 Then Elements.ForEach(Sub(e As YouTubeMediaContainerBase) e.AbsolutePath = ap)
End Set
End Property
Public Overridable Property File As SFile Implements IYouTubeMediaContainer.File
Get
Return _File
@@ -550,11 +560,16 @@ Namespace API.YouTube.Objects
Set(ByVal f As SFile)
Select Case ObjectType
Case YouTubeMediaType.Channel : _File = f.Path
Case YouTubeMediaType.PlayList : _File.Path = $"{f.PathWithSeparator}{GetPlayListTitle()}"
Case YouTubeMediaType.PlayList
If AbsolutePath Then
_File.Path = f.Path
Else
_File.Path = $"{f.PathWithSeparator}{GetPlayListTitle()}"
End If
Case YouTubeMediaType.Single
If PlaylistCount > 0 And Not FileIgnorePlaylist Then
_File.Path = f.Path
Dim pls$ = GetPlayListTitle()
Dim pls$ = If(AbsolutePath, String.Empty, GetPlayListTitle())
If Not _File.Path.Contains(pls) Then _File.Path = $"{_File.PathWithSeparator(Not pls.IsEmptyString)}{pls}"
ElseIf Not f.Name.IsEmptyString Then
_File = f

2
SCrawler.YouTubeDownloader/.editorconfig
View File

@@ -1,3 +1,3 @@
[*.vb]
# Modifier preferences
file_header_template = Copyright (C) 2023 Andy https://github.com/AAndyProgram\nThis program is free software: you can redistribute it and/or modify\nit under the terms of the GNU General Public License as published by\nthe Free Software Foundation, either version 3 of the License, or\n(at your option) any later version.\n\nThis program is distributed in the hope that it will be useful,\nbut WITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\nGNU General Public License for more details.\n\nYou should have received a copy of the GNU General Public License\nalong with this program. If not, see <https://www.gnu.org/licenses/>
file_header_template = Copyright (C) Andy https://github.com/AAndyProgram\nThis program is free software: you can redistribute it and/or modify\nit under the terms of the GNU General Public License as published by\nthe Free Software Foundation, either version 3 of the License, or\n(at your option) any later version.\n\nThis program is distributed in the hope that it will be useful,\nbut WITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\nGNU General Public License for more details.\n\nYou should have received a copy of the GNU General Public License\nalong with this program. If not, see <https://www.gnu.org/licenses/>

2
SCrawler/.editorconfig
View File

@@ -123,4 +123,4 @@ insert_final_newline=false
[*.vb]
# Modifier preferences
visual_basic_preferred_modifier_order = Partial,Default,Private,Protected,Public,Friend,NotOverridable,Overridable,MustOverride,Overloads,Overrides,MustInherit,NotInheritable,Static,Shared,Shadows,ReadOnly,WriteOnly,Dim,Const,WithEvents,Widening,Narrowing,Custom,Async:suggestion
file_header_template = Copyright (C) 2023 Andy https://github.com/AAndyProgram\nThis program is free software: you can redistribute it and/or modify\nit under the terms of the GNU General Public License as published by\nthe Free Software Foundation, either version 3 of the License, or\n(at your option) any later version.\n\nThis program is distributed in the hope that it will be useful,\nbut WITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\nGNU General Public License for more details.\n\nYou should have received a copy of the GNU General Public License\nalong with this program. If not, see <https://www.gnu.org/licenses/>
file_header_template = Copyright (C) Andy https://github.com/AAndyProgram\nThis program is free software: you can redistribute it and/or modify\nit under the terms of the GNU General Public License as published by\nthe Free Software Foundation, either version 3 of the License, or\n(at your option) any later version.\n\nThis program is distributed in the hope that it will be useful,\nbut WITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\nGNU General Public License for more details.\n\nYou should have received a copy of the GNU General Public License\nalong with this program. If not, see <https://www.gnu.org/licenses/>

9
SCrawler/API/Base/SiteSettingsBase.vb
View File

@@ -10,6 +10,7 @@ Imports System.Reflection
Imports SCrawler.Plugin
Imports SCrawler.Plugin.Attributes
Imports PersonalUtilities.Tools
Imports PersonalUtilities.Tools.Web.Cookies
Imports PersonalUtilities.Tools.Web.Clients
Imports PersonalUtilities.Functions.RegularExpressions
Imports Download = SCrawler.Plugin.ISiteSettings.Download
@@ -73,7 +74,7 @@ Namespace API.Base
Responser.Cookies.ChangedAllowInternalDrop = Not _UseNetscapeCookies
Responser.Cookies.Changed = False
End If
If b And _UseNetscapeCookies Then Update_SaveCookiesNetscape()
If b AndAlso _UseNetscapeCookies AndAlso Not CookiesNetscapeFile.Exists Then Update_SaveCookiesNetscape()
End Set
End Property
Private Property IResponserContainer_Responser As Responser Implements IResponserContainer.Responser
@@ -140,6 +141,12 @@ Namespace API.Base
Protected _SiteEditorFormOpened As Boolean = False
Friend Overridable Sub BeginEdit() Implements ISiteSettings.BeginEdit
_SiteEditorFormOpened = True
If UseNetscapeCookies And CookiesNetscapeFile.Exists Then
With Responser.Cookies
.Clear()
.AddRange(CookieKeeper.ParseNetscapeText(CookiesNetscapeFile.GetText, EDP.SendToLog + EDP.ReturnValue),, EDP.None)
End With
End If
End Sub
Friend Overridable Sub EndEdit() Implements ISiteSettings.EndEdit
If _SiteEditorFormOpened Then DomainsReset()

3
SCrawler/API/Base/UserDataBase.vb
View File

@@ -1942,6 +1942,7 @@ BlockNullPicture:
Try
Dim f As SFile
Dim v As Boolean = IsVirtual
Settings.Feeds.Load()
If IncludedInCollection And __CollectionName.IsEmptyString And __SpecialCollectionPath.IsEmptyString Then
Settings.Users.Add(Me)
@@ -1984,6 +1985,7 @@ BlockNullPicture:
Settings.UsersList.Remove(UserBefore)
Settings.UpdateUsersList(User)
Settings.Feeds.UpdateUsers(UserBefore, User)
UpdateUserInformation()
Return True
Catch ex As Exception
@@ -2037,6 +2039,7 @@ BlockNullPicture:
End If
If Not ScriptData.IsEmptyString AndAlso ScriptData.Contains(UserBefore.File.PathNoSeparator) Then _
ScriptData = ScriptData.Replace(UserBefore.File.PathNoSeparator, MyFile.PathNoSeparator)
Settings.Feeds.UpdateUsers(UserBefore, User)
UpdateUserInformation()
End If
Catch ioex As InvalidOperationException When ioex.HelpLink = 1

15
SCrawler/API/TikTok/SiteSettings.vb
View File

@@ -14,20 +14,24 @@ Namespace API.TikTok
<Manifest("AndyProgram_TikTok"), SpecialForm(False), SeparatedTasks(1)>
Friend Class SiteSettings : Inherits SiteSettingsBase
<PropertyOption(ControlText:="Remove tags from title"), PXML, PClonable>
Friend Property RemoveTagsFromTitle As PropertyValue
Friend ReadOnly Property RemoveTagsFromTitle As PropertyValue
<PropertyOption(ControlText:="Use native title", ControlToolTip:="Use a user-created video title for the filename instead of the video ID."), PXML, PClonable>
Friend Property TitleUseNative As PropertyValue
Friend ReadOnly Property TitleUseNative As PropertyValue
<PropertyOption(ControlText:="Use native title in standalone downloader",
ControlToolTip:="Use a user-created video title for the filename instead of the video ID."), PXML, PClonable>
Friend Property TitleUseNativeSTD As PropertyValue
Friend ReadOnly Property TitleUseNativeSTD As PropertyValue
<PropertyOption(ControlText:="Add video ID to video title"), PXML, PClonable>
Friend Property TitleAddVideoID As PropertyValue
Friend ReadOnly Property TitleAddVideoID As PropertyValue
<PropertyOption(ControlText:="Use video date as file date",
ControlToolTip:="Set the file date to the date the video was added (website) (if available)."), PXML, PClonable>
Friend ReadOnly Property UseParsedVideoDate As PropertyValue
Friend Sub New(ByVal AccName As String, ByVal Temp As Boolean)
MyBase.New("TikTok", "www.tiktok.com", AccName, Temp, My.Resources.SiteResources.TikTokIcon_32, My.Resources.SiteResources.TikTokPic_192)
RemoveTagsFromTitle = New PropertyValue(False)
TitleUseNative = New PropertyValue(True)
TitleUseNativeSTD = New PropertyValue(False)
TitleAddVideoID = New PropertyValue(True)
UseParsedVideoDate = New PropertyValue(True)
UseNetscapeCookies = True
UrlPatternUser = "https://www.tiktok.com/@{0}/"
UserRegex = RParams.DMS("[htps:/]{7,8}.*?tiktok.com/@([^/]+)", 1)
@@ -45,5 +49,8 @@ Namespace API.TikTok
Using f As New InternalSettingsForm(Options, Me, False) : f.ShowDialog() : End Using
End If
End Sub
Friend Overrides Function GetUserUrl(ByVal User As IPluginContentProvider) As String
Return String.Format(UrlPatternUser, DirectCast(User, UserData).TrueName)
End Function
End Class
End Namespace

31
SCrawler/API/TikTok/UserData.vb
View File

@@ -48,6 +48,15 @@ Namespace API.TikTok
Friend Property TitleUseNative As Boolean = True
Friend Property TitleAddVideoID As Boolean = True
Private Property LastDownloadDate As Date? = Nothing
Private _TrueName As String = String.Empty
Friend Property TrueName As String
Get
Return _TrueName.IfNullOrEmpty(Name)
End Get
Set(ByVal NewName As String)
_TrueName = NewName
End Set
End Property
#End Region
#Region "Exchange"
Friend Overrides Function ExchangeOptionsGet() As Object
@@ -72,11 +81,13 @@ Namespace API.TikTok
TitleAddVideoID = .Value(Name_TitleAddVideoID).FromXML(Of Boolean)(True)
LastDownloadDate = AConvert(Of Date)(.Value(Name_LastDownloadDate), ADateTime.Formats.BaseDateTime, Nothing)
If Not LastDownloadDate.HasValue Then LastDownloadDate = LastUpdated
_TrueName = .Value(Name_TrueName)
Else
.Add(Name_RemoveTagsFromTitle, RemoveTagsFromTitle.BoolToInteger)
.Add(Name_TitleUseNative, TitleUseNative.BoolToInteger)
.Add(Name_TitleAddVideoID, TitleAddVideoID.BoolToInteger)
.Add(Name_LastDownloadDate, AConvert(Of String)(LastDownloadDate, AModes.XML, ADateTime.Formats.BaseDateTime, String.Empty))
.Add(Name_TrueName, _TrueName)
End If
End With
End Sub
@@ -89,14 +100,15 @@ Namespace API.TikTok
#End Region
#Region "Download functions"
Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
Dim URL$ = $"https://www.tiktok.com/@{Name}"
Dim URL$ = $"https://www.tiktok.com/@{TrueName}"
Using cache As CacheKeeper = CreateCache()
Try
Dim postID$, title$, postUrl$
Dim postID$, title$, postUrl$, newName$
Dim postDate As Date?
Dim dateAfterC As Date? = Nothing
Dim dateBefore As Date? = DownloadDateTo
Dim dateAfter As Date? = DownloadDateFrom
Dim baseDataObtained As Boolean = False
If _ContentList.Count > 0 Then
With (From d In _ContentList Where d.Post.Date.HasValue Select d.Post.Date.Value)
@@ -138,6 +150,20 @@ Namespace API.TikTok
j = JsonDocument.Parse(file.GetText, EDP.ReturnValue)
If j.ListExists Then
If j.Value("_type").StringToLower = "video" Then
If Not baseDataObtained Then
baseDataObtained = True
If ID.IsEmptyString Then
ID = j.Value("uploader_id")
If Not ID.IsEmptyString Then _ForceSaveUserInfo = True
End If
newName = j.Value("uploader")
If Not newName.IsEmptyString Then
If Not _TrueName = newName Then _ForceSaveUserInfo = True
_TrueName = newName
End If
newName = j.Value("creator")
If Not newName.IsEmptyString Then UserSiteName = newName
End If
postID = j.Value("id")
If Not _TempPostsList.Contains(postID) Then
_TempPostsList.Add(postID)
@@ -211,6 +237,7 @@ Namespace API.TikTok
End If
If DateBefore.HasValue Then command &= $"--datebefore {DateBefore.Value.AddDays(1).ToStringDate(SimpleDateConverter)} "
If DateAfter.HasValue Then command &= $"--dateafter {DateAfter.Value.AddDays(-1).ToStringDate(SimpleDateConverter)} "
If Not CBool(MySettings.UseParsedVideoDate.Value) Then command &= "--no-mtime "
If MySettings.CookiesNetscapeFile.Exists Then command &= $"--no-cookies-from-browser --cookies ""{MySettings.CookiesNetscapeFile}"" "
command &= $"{URL} "
If SupportOutput Then

1
SCrawler/API/YouTube/SiteSettings.vb
View File

@@ -32,6 +32,7 @@ Namespace API.YouTube
DownloadPlaylists = New PropertyValue(False)
UseCookies = New PropertyValue(False)
_SubscriptionsAllowed = True
UseNetscapeCookies = True
End Sub
#End Region
#Region "GetInstance"

21
SCrawler/Download/Automation/AutoDownloader.vb
View File

@@ -184,6 +184,7 @@ Namespace DownloadObjects
#Region "XML Names"
Private Const Name_Mode As String = "Mode"
Private Const Name_Groups As String = "Groups"
Private Const Name_IsManual As String = "IsManual"
Private Const Name_Timer As String = "Timer"
Private Const Name_StartupDelay As String = "StartupDelay"
Private Const Name_LastDownloadDate As String = "LastDownloadDate"
@@ -205,6 +206,7 @@ Namespace DownloadObjects
End Set
End Property
Friend ReadOnly Property Groups As List(Of String)
Friend Property IsManual As Boolean = False
Friend Property Timer As Integer = DefaultTimer
Friend Property StartupDelay As Integer = 1
Friend Property ShowNotifications As Boolean = True
@@ -281,7 +283,11 @@ Namespace DownloadObjects
Return OutStr
End Function
Public Overrides Function ToString() As String
If IsManual Then
Return $"{Name} (manual): last download date: {GetLastDateString()}"
Else
Return $"{Name} ({GetWorkingState()}): last download date: {GetLastDateString()}; next run: {GetNextDateString()}"
End If
End Function
#End Region
#End Region
@@ -307,6 +313,7 @@ Namespace DownloadObjects
If Name.IsEmptyString Then Name = "Default"
Groups.ListAddList(x.Value(Name_Groups).StringToList(Of String)("|"), LAP.NotContainsOnly)
IsManual = x.Value(Name_IsManual).FromXML(Of Boolean)(False)
Timer = x.Value(Name_Timer).FromXML(Of Integer)(DefaultTimer)
If Timer <= 0 Then Timer = DefaultTimer
StartupDelay = x.Value(Name_StartupDelay).FromXML(Of Integer)(0)
@@ -364,6 +371,7 @@ Namespace DownloadObjects
Return Export(New EContainer(Scheduler.Name_Plan, String.Empty) From {
New EContainer(Name_Mode, CInt(Mode)),
New EContainer(Name_Groups, Groups.ListToString("|")),
New EContainer(Name_IsManual, IsManual.BoolToInteger),
New EContainer(Name_Timer, Timer),
New EContainer(Name_StartupDelay, StartupDelay),
New EContainer(Name_ShowNotifications, ShowNotifications.BoolToInteger),
@@ -383,7 +391,8 @@ Namespace DownloadObjects
End Get
End Property
Private _StartTime As Date = Now
Friend Sub Start(ByVal Init As Boolean)
Friend Sub Start(ByVal Init As Boolean, Optional ByVal Force As Boolean = False)
If Not IsManual Or Force Then
If Init Then _StartTime = Now
_IsNewPlan = False
If Not Working And Not Mode = Modes.None Then
@@ -391,6 +400,7 @@ Namespace DownloadObjects
AThread.SetApartmentState(ApartmentState.MTA)
AThread.Start()
End If
End If
End Sub
Private _StopRequested As Boolean = False
Private _Pause As PauseModes = PauseModes.Disabled
@@ -456,6 +466,7 @@ Namespace DownloadObjects
End Sub
Friend Sub ForceStart()
_ForceStartRequested = True
If IsManual Then Start(False, True)
End Sub
Private _ForceStartRequested As Boolean = False
Private _SpecialDelayUse As Boolean = False
@@ -464,7 +475,8 @@ Namespace DownloadObjects
Try
Dim _StartDownload As Boolean
While (Not _StopRequested Or Downloader.Working) And Not Mode = Modes.None
If ((NextExecutionDate < Now And Not IsPaused) Or _ForceStartRequested) And Not _StopRequested And Not Mode = Modes.None Then
If ((IsManual And _ForceStartRequested) Or (NextExecutionDate < Now And Not IsPaused) Or _ForceStartRequested) And
Not _StopRequested And Not Mode = Modes.None Then
If Downloader.Working Then
_SpecialDelayUse = True
Else
@@ -478,7 +490,10 @@ Namespace DownloadObjects
Else
_StartDownload = NextExecutionDate.AddMilliseconds(1000 * (Index + 1)).Ticks <= Now.Ticks
End If
If _StartDownload Then Download()
If _StartDownload Then
Download()
If IsManual Then Exit While
End If
End If
End If
End If

81
SCrawler/Download/Automation/AutoDownloaderEditorForm.Designer.vb generated
View File

@@ -25,12 +25,12 @@ Namespace DownloadObjects
Me.components = New System.ComponentModel.Container()
Dim CONTAINER_MAIN As System.Windows.Forms.ToolStripContainer
Dim TP_MODE As System.Windows.Forms.TableLayoutPanel
Dim ActionButton1 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim ActionButton5 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim resources As System.ComponentModel.ComponentResourceManager = New System.ComponentModel.ComponentResourceManager(GetType(AutoDownloaderEditorForm))
Dim ActionButton2 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim ActionButton6 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim TP_NOTIFY As System.Windows.Forms.TableLayoutPanel
Dim ActionButton3 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim ActionButton4 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim ActionButton7 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim ActionButton8 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Dim TT_MAIN As System.Windows.Forms.ToolTip
Me.DEF_GROUP = New SCrawler.DownloadObjects.Groups.GroupDefaults()
Me.OPT_ALL = New System.Windows.Forms.RadioButton()
@@ -46,6 +46,7 @@ Namespace DownloadObjects
Me.TXT_TIMER = New PersonalUtilities.Forms.Controls.TextBoxExtended()
Me.NUM_DELAY = New PersonalUtilities.Forms.Controls.TextBoxExtended()
Me.LBL_LAST_TIME_UP = New System.Windows.Forms.Label()
Me.CH_MANUAL = New System.Windows.Forms.CheckBox()
CONTAINER_MAIN = New System.Windows.Forms.ToolStripContainer()
TP_MODE = New System.Windows.Forms.TableLayoutPanel()
TP_NOTIFY = New System.Windows.Forms.TableLayoutPanel()
@@ -72,7 +73,7 @@ Namespace DownloadObjects
CONTAINER_MAIN.Location = New System.Drawing.Point(0, 0)
CONTAINER_MAIN.Name = "CONTAINER_MAIN"
CONTAINER_MAIN.RightToolStripPanelVisible = False
CONTAINER_MAIN.Size = New System.Drawing.Size(476, 388)
CONTAINER_MAIN.Size = New System.Drawing.Size(476, 413)
CONTAINER_MAIN.TabIndex = 0
CONTAINER_MAIN.TopToolStripPanelVisible = False
'
@@ -84,13 +85,14 @@ Namespace DownloadObjects
Me.DEF_GROUP.Controls.Add(TP_MODE, 0, 0)
Me.DEF_GROUP.Controls.Add(Me.TXT_GROUPS, 0, 8)
Me.DEF_GROUP.Controls.Add(TP_NOTIFY, 0, 9)
Me.DEF_GROUP.Controls.Add(Me.TXT_TIMER, 0, 10)
Me.DEF_GROUP.Controls.Add(Me.NUM_DELAY, 0, 11)
Me.DEF_GROUP.Controls.Add(Me.LBL_LAST_TIME_UP, 0, 12)
Me.DEF_GROUP.Controls.Add(Me.TXT_TIMER, 0, 11)
Me.DEF_GROUP.Controls.Add(Me.NUM_DELAY, 0, 12)
Me.DEF_GROUP.Controls.Add(Me.LBL_LAST_TIME_UP, 0, 13)
Me.DEF_GROUP.Controls.Add(Me.CH_MANUAL, 0, 10)
Me.DEF_GROUP.Dock = System.Windows.Forms.DockStyle.Fill
Me.DEF_GROUP.Location = New System.Drawing.Point(0, 0)
Me.DEF_GROUP.Name = "DEF_GROUP"
Me.DEF_GROUP.RowCount = 14
Me.DEF_GROUP.RowCount = 15
Me.DEF_GROUP.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEF_GROUP.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.DEF_GROUP.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
@@ -101,11 +103,11 @@ Namespace DownloadObjects
Me.DEF_GROUP.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.DEF_GROUP.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.DEF_GROUP.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.DEF_GROUP.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEF_GROUP.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.DEF_GROUP.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
Me.DEF_GROUP.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 25.0!))
Me.DEF_GROUP.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
Me.DEF_GROUP.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 20.0!))
Me.DEF_GROUP.Size = New System.Drawing.Size(476, 388)
Me.DEF_GROUP.TabIndex = 0
'
@@ -199,16 +201,15 @@ Namespace DownloadObjects
'
'TXT_GROUPS
'
ActionButton1.BackgroundImage = CType(resources.GetObject("ActionButton1.BackgroundImage"), System.Drawing.Image)
ActionButton1.Name = "Edit"
ActionButton2.BackgroundImage = CType(resources.GetObject("ActionButton2.BackgroundImage"), System.Drawing.Image)
ActionButton2.Name = "Clear"
Me.TXT_GROUPS.Buttons.Add(ActionButton1)
Me.TXT_GROUPS.Buttons.Add(ActionButton2)
ActionButton5.BackgroundImage = CType(resources.GetObject("ActionButton5.BackgroundImage"), System.Drawing.Image)
ActionButton5.Name = "Edit"
ActionButton6.BackgroundImage = CType(resources.GetObject("ActionButton6.BackgroundImage"), System.Drawing.Image)
ActionButton6.Name = "Clear"
Me.TXT_GROUPS.Buttons.Add(ActionButton5)
Me.TXT_GROUPS.Buttons.Add(ActionButton6)
Me.TXT_GROUPS.CaptionText = "Groups"
Me.TXT_GROUPS.CaptionWidth = 50.0R
Me.TXT_GROUPS.Dock = System.Windows.Forms.DockStyle.Fill
Me.TXT_GROUPS.Lines = New String(-1) {}
Me.TXT_GROUPS.Location = New System.Drawing.Point(4, 224)
Me.TXT_GROUPS.Name = "TXT_GROUPS"
Me.TXT_GROUPS.Size = New System.Drawing.Size(468, 22)
@@ -285,25 +286,24 @@ Namespace DownloadObjects
'
'TXT_TIMER
'
ActionButton3.BackgroundImage = CType(resources.GetObject("ActionButton3.BackgroundImage"), System.Drawing.Image)
ActionButton3.Name = "Refresh"
Me.TXT_TIMER.Buttons.Add(ActionButton3)
ActionButton7.BackgroundImage = CType(resources.GetObject("ActionButton7.BackgroundImage"), System.Drawing.Image)
ActionButton7.Name = "Refresh"
Me.TXT_TIMER.Buttons.Add(ActionButton7)
Me.TXT_TIMER.CaptionText = "Timer"
Me.TXT_TIMER.CaptionToolTipEnabled = True
Me.TXT_TIMER.CaptionToolTipText = "Timer (in minutes)"
Me.TXT_TIMER.CaptionWidth = 50.0R
Me.TXT_TIMER.Dock = System.Windows.Forms.DockStyle.Fill
Me.TXT_TIMER.Lines = New String(-1) {}
Me.TXT_TIMER.Location = New System.Drawing.Point(4, 282)
Me.TXT_TIMER.Location = New System.Drawing.Point(4, 308)
Me.TXT_TIMER.Name = "TXT_TIMER"
Me.TXT_TIMER.Size = New System.Drawing.Size(468, 22)
Me.TXT_TIMER.TabIndex = 3
Me.TXT_TIMER.TabIndex = 4
'
'NUM_DELAY
'
ActionButton4.BackgroundImage = CType(resources.GetObject("ActionButton4.BackgroundImage"), System.Drawing.Image)
ActionButton4.Name = "Refresh"
Me.NUM_DELAY.Buttons.Add(ActionButton4)
ActionButton8.BackgroundImage = CType(resources.GetObject("ActionButton8.BackgroundImage"), System.Drawing.Image)
ActionButton8.Name = "Refresh"
Me.NUM_DELAY.Buttons.Add(ActionButton8)
Me.NUM_DELAY.CaptionText = "Delay"
Me.NUM_DELAY.CaptionToolTipEnabled = True
Me.NUM_DELAY.CaptionToolTipText = "Startup delay"
@@ -311,13 +311,12 @@ Namespace DownloadObjects
Me.NUM_DELAY.ClearTextByButtonClear = False
Me.NUM_DELAY.ControlMode = PersonalUtilities.Forms.Controls.TextBoxExtended.ControlModes.NumericUpDown
Me.NUM_DELAY.Dock = System.Windows.Forms.DockStyle.Fill
Me.NUM_DELAY.Lines = New String(-1) {}
Me.NUM_DELAY.Location = New System.Drawing.Point(4, 311)
Me.NUM_DELAY.Location = New System.Drawing.Point(4, 337)
Me.NUM_DELAY.Name = "NUM_DELAY"
Me.NUM_DELAY.NumberMaximum = New Decimal(New Integer() {1440, 0, 0, 0})
Me.NUM_DELAY.NumberUpDownAlign = System.Windows.Forms.LeftRightAlignment.Left
Me.NUM_DELAY.Size = New System.Drawing.Size(468, 22)
Me.NUM_DELAY.TabIndex = 4
Me.NUM_DELAY.TabIndex = 5
Me.NUM_DELAY.Text = "0"
'
'LBL_LAST_TIME_UP
@@ -325,26 +324,39 @@ Namespace DownloadObjects
Me.LBL_LAST_TIME_UP.AutoSize = True
Me.LBL_LAST_TIME_UP.Dock = System.Windows.Forms.DockStyle.Fill
Me.LBL_LAST_TIME_UP.Font = New System.Drawing.Font("Microsoft Sans Serif", 8.25!, System.Drawing.FontStyle.Italic, System.Drawing.GraphicsUnit.Point, CType(204, Byte))
Me.LBL_LAST_TIME_UP.Location = New System.Drawing.Point(4, 337)
Me.LBL_LAST_TIME_UP.Location = New System.Drawing.Point(4, 363)
Me.LBL_LAST_TIME_UP.Name = "LBL_LAST_TIME_UP"
Me.LBL_LAST_TIME_UP.Size = New System.Drawing.Size(468, 25)
Me.LBL_LAST_TIME_UP.TabIndex = 5
Me.LBL_LAST_TIME_UP.TabIndex = 6
Me.LBL_LAST_TIME_UP.Text = "Last download date: "
Me.LBL_LAST_TIME_UP.TextAlign = System.Drawing.ContentAlignment.TopCenter
'
'CH_MANUAL
'
Me.CH_MANUAL.AutoSize = True
Me.CH_MANUAL.Dock = System.Windows.Forms.DockStyle.Fill
Me.CH_MANUAL.Location = New System.Drawing.Point(4, 282)
Me.CH_MANUAL.Name = "CH_MANUAL"
Me.CH_MANUAL.Size = New System.Drawing.Size(468, 19)
Me.CH_MANUAL.TabIndex = 3
Me.CH_MANUAL.Text = "Run this task manually"
TT_MAIN.SetToolTip(Me.CH_MANUAL, "If this checkbox is selected, this task can only be started manually (using the '" &
"Start (force)' button)")
Me.CH_MANUAL.UseVisualStyleBackColor = True
'
'AutoDownloaderEditorForm
'
Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!)
Me.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font
Me.ClientSize = New System.Drawing.Size(476, 388)
Me.ClientSize = New System.Drawing.Size(476, 413)
Me.Controls.Add(CONTAINER_MAIN)
Me.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedSingle
Me.Icon = Global.SCrawler.My.Resources.Resources.ArrowDownIcon_Blue_24
Me.KeyPreview = True
Me.MaximizeBox = False
Me.MaximumSize = New System.Drawing.Size(492, 427)
Me.MaximumSize = New System.Drawing.Size(492, 452)
Me.MinimizeBox = False
Me.MinimumSize = New System.Drawing.Size(492, 427)
Me.MinimumSize = New System.Drawing.Size(492, 452)
Me.Name = "AutoDownloaderEditorForm"
Me.ShowInTaskbar = False
Me.SizeGripStyle = System.Windows.Forms.SizeGripStyle.Hide
@@ -378,5 +390,6 @@ Namespace DownloadObjects
Private WithEvents CH_SHOW_PIC As CheckBox
Private WithEvents CH_SHOW_PIC_USER As CheckBox
Private WithEvents CH_NOTIFY_SIMPLE As CheckBox
Private WithEvents CH_MANUAL As CheckBox
End Class
End Namespace

86
SCrawler/Download/Automation/AutoDownloaderEditorForm.resx
View File

@@ -123,14 +123,8 @@
<metadata name="TP_MODE.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<value>False</value>
</metadata>
<metadata name="TT_MAIN.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<value>False</value>
</metadata>
<metadata name="TT_MAIN.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<value>17, 17</value>
</metadata>
<assembly alias="System.Drawing" name="System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a" />
<data name="ActionButton1.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<data name="ActionButton5.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAACH
DwAAjA8AAP1SAACBQAAAfXkAAOmLAAA85QAAGcxzPIV3AAAKOWlDQ1BQaG90b3Nob3AgSUNDIHByb2Zp
@@ -188,7 +182,7 @@
AAAASUVORK5CYII=
</value>
</data>
<data name="ActionButton2.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<data name="ActionButton6.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGPC/xhBQAAAAlwSFlzAAAO
xAAADsQBlSsOGwAAAIZJREFUOE+1j10KwCAMgz2b755xl/IsvnaL2K20UfbDAmEako+ZROSTafjE12Go
@@ -199,41 +193,53 @@
<metadata name="TP_NOTIFY.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<value>False</value>
</metadata>
<data name="ActionButton7.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6
JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAAACXBIWXMAAAsTAAALEwEAmpwYAAACOElE
QVQ4T2P4//8/QczOJyyqHpzfiE0OQwAZC8iqszAzs7CJ69o4BR768V/W2jcGXQ0KB4aFNS3dDQtnrbCb
ePCK48wTN1wXXXzge/jXf/clV55zC4hIIatF0cjIyMikElzc57z0wX+XHd/+2+//99/ywP//xlu//tdb
+eK/4Zp3/1WTOhYzARViNUAluKjTdf37/0ZTTn9TbdhwXblhwwW1/qOP1Ja9+K8w+95/6cm3/6v2Xvkv
qKjniGGAoIqRpW3/4e8S9uGdzFz82gwMDFxAzCxm4ZegtuLDf+VJ1/8rZM25IqLvnM/CximCYYCic1QN
v7x2JIwPwyrJ3XNUylddE9G2TWNmZOBDl4czmJiZMSRBmFdSyYyJgUEQmxwIYxWEYXZBCUls4sgYq6CA
prWNbtG8nXKeaVPR5XiVjSxEzf0yYXy4BBMLO6eQjoOXZvrkbbazrv53Xf/2v4CSbjBMXkhBl1/CMyNZ
qWnvGy5pNQ+YONwAfjXzAOupl/47LLr333L50/96q9/8l23YdES6cO5KuYqVW+R7Tj6SnfP0v4hryjyY
HhQDmFjYeHVKFp7WX/Xuv9Kq9/+Vd/z7r7rv/3+l7f//y676DEwDN/9L+BVvYkKLCTgDhNkkVUyVlr74
qbbz73/VOTc/qsy89kWx+9h7qbQpJwS1bbOAscGGrB6EUTggLOqf16C55ft/HlnNAFZOXgVWdi4FRgYG
VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg==
</value>
</data>
<data name="ActionButton8.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6
JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAAACXBIWXMAAAsTAAALEwEAmpwYAAACOElE
QVQ4T2P4//8/QczOJyyqHpzfiE0OQwAZC8iqszAzs7CJ69o4BR768V/W2jcGXQ0KB4aFNS3dDQtnrbCb
ePCK48wTN1wXXXzge/jXf/clV55zC4hIIatF0cjIyMikElzc57z0wX+XHd/+2+//99/ywP//xlu//tdb
+eK/4Zp3/1WTOhYzARViNUAluKjTdf37/0ZTTn9TbdhwXblhwwW1/qOP1Ja9+K8w+95/6cm3/6v2Xvkv
qKjniGGAoIqRpW3/4e8S9uGdzFz82gwMDFxAzCxm4ZegtuLDf+VJ1/8rZM25IqLvnM/CximCYYCic1QN
v7x2JIwPwyrJ3XNUylddE9G2TWNmZOBDl4czmJiZMSRBmFdSyYyJgUEQmxwIYxWEYXZBCUls4sgYq6CA
prWNbtG8nXKeaVPR5XiVjSxEzf0yYXy4BBMLO6eQjoOXZvrkbbazrv53Xf/2v4CSbjBMXkhBl1/CMyNZ
qWnvGy5pNQ+YONwAfjXzAOupl/47LLr333L50/96q9/8l23YdES6cO5KuYqVW+R7Tj6SnfP0v4hryjyY
HhQDmFjYeHVKFp7WX/Xuv9Kq9/+Vd/z7r7rv/3+l7f//y676DEwDN/9L+BVvYkKLCTgDhNkkVUyVlr74
qbbz73/VOTc/qsy89kWx+9h7qbQpJwS1bbOAscGGrB6EUTggLOqf16C55ft/HlnNAFZOXgVWdi4FRgYG
VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg==
</value>
</data>
<metadata name="TT_MAIN.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<value>False</value>
</metadata>
<metadata name="TT_MAIN.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<value>17, 17</value>
</metadata>
<metadata name="TT_MAIN.GenerateMember" type="System.Boolean, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089">
<value>False</value>
</metadata>
<metadata name="TT_MAIN.TrayLocation" type="System.Drawing.Point, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
<value>17, 17</value>
</metadata>
<data name="CH_NOTIFY_SIMPLE.ToolTip" xml:space="preserve">
<value>Show a simple notification instead of a user notification.
This means that if any user data has been downloaded with the plan, a simple notification will be shown with the number of users downloaded.
The 'Image' and 'User icon' parameters will be ignored.</value>
</data>
<data name="ActionButton3.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6
JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAAACXBIWXMAAAsTAAALEwEAmpwYAAACOElE
QVQ4T2P4//8/QczOJyyqHpzfiE0OQwAZC8iqszAzs7CJ69o4BR768V/W2jcGXQ0KB4aFNS3dDQtnrbCb
ePCK48wTN1wXXXzge/jXf/clV55zC4hIIatF0cjIyMikElzc57z0wX+XHd/+2+//99/ywP//xlu//tdb
+eK/4Zp3/1WTOhYzARViNUAluKjTdf37/0ZTTn9TbdhwXblhwwW1/qOP1Ja9+K8w+95/6cm3/6v2Xvkv
qKjniGGAoIqRpW3/4e8S9uGdzFz82gwMDFxAzCxm4ZegtuLDf+VJ1/8rZM25IqLvnM/CximCYYCic1QN
v7x2JIwPwyrJ3XNUylddE9G2TWNmZOBDl4czmJiZMSRBmFdSyYyJgUEQmxwIYxWEYXZBCUls4sgYq6CA
prWNbtG8nXKeaVPR5XiVjSxEzf0yYXy4BBMLO6eQjoOXZvrkbbazrv53Xf/2v4CSbjBMXkhBl1/CMyNZ
qWnvGy5pNQ+YONwAfjXzAOupl/47LLr333L50/96q9/8l23YdES6cO5KuYqVW+R7Tj6SnfP0v4hryjyY
HhQDmFjYeHVKFp7WX/Xuv9Kq9/+Vd/z7r7rv/3+l7f//y676DEwDN/9L+BVvYkKLCTgDhNkkVUyVlr74
qbbz73/VOTc/qsy89kWx+9h7qbQpJwS1bbOAscGGrB6EUTggLOqf16C55ft/HlnNAFZOXgVWdi4FRgYG
VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg==
</value>
</data>
<data name="ActionButton4.BackgroundImage" type="System.Drawing.Bitmap, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>
iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAABGdBTUEAALGOfPtRkwAAACBjSFJNAAB6
JQAAgIMAAPn/AACA6QAAdTAAAOpgAAA6mAAAF2+SX8VGAAAACXBIWXMAAAsTAAALEwEAmpwYAAACOElE
QVQ4T2P4//8/QczOJyyqHpzfiE0OQwAZC8iqszAzs7CJ69o4BR768V/W2jcGXQ0KB4aFNS3dDQtnrbCb
ePCK48wTN1wXXXzge/jXf/clV55zC4hIIatF0cjIyMikElzc57z0wX+XHd/+2+//99/ywP//xlu//tdb
+eK/4Zp3/1WTOhYzARViNUAluKjTdf37/0ZTTn9TbdhwXblhwwW1/qOP1Ja9+K8w+95/6cm3/6v2Xvkv
qKjniGGAoIqRpW3/4e8S9uGdzFz82gwMDFxAzCxm4ZegtuLDf+VJ1/8rZM25IqLvnM/CximCYYCic1QN
v7x2JIwPwyrJ3XNUylddE9G2TWNmZOBDl4czmJiZMSRBmFdSyYyJgUEQmxwIYxWEYXZBCUls4sgYq6CA
prWNbtG8nXKeaVPR5XiVjSxEzf0yYXy4BBMLO6eQjoOXZvrkbbazrv53Xf/2v4CSbjBMXkhBl1/CMyNZ
qWnvGy5pNQ+YONwAfjXzAOupl/47LLr333L50/96q9/8l23YdES6cO5KuYqVW+R7Tj6SnfP0v4hryjyY
HhQDmFjYeHVKFp7WX/Xuv9Kq9/+Vd/z7r7rv/3+l7f//y676DEwDN/9L+BVvYkKLCTgDhNkkVUyVlr74
qbbz73/VOTc/qsy89kWx+9h7qbQpJwS1bbOAscGGrB6EUTggLOqf16C55ft/HlnNAFZOXgVWdi4FRgYG
VnR1MIwhwMTCyqEQ37qEmZVDFF0OE/9nAACtFF4Ey6OP+wAAAABJRU5ErkJggg==
</value>
</data>
</root>

3
SCrawler/Download/Automation/AutoDownloaderEditorForm.vb
View File

@@ -58,6 +58,7 @@ Namespace DownloadObjects
CH_NOTIFY_SIMPLE.Checked = .ShowSimpleNotification
CH_SHOW_PIC.Checked = .ShowPictureDownloaded
CH_SHOW_PIC_USER.Checked = .ShowPictureUser
CH_MANUAL.Checked = .IsManual
TXT_TIMER.Text = .Timer
NUM_DELAY.Value = .StartupDelay
LBL_LAST_TIME_UP.Text = .Information
@@ -94,8 +95,10 @@ Namespace DownloadObjects
.ShowSimpleNotification = CH_NOTIFY_SIMPLE.Checked
.ShowPictureDownloaded = CH_SHOW_PIC.Checked
.ShowPictureUser = CH_SHOW_PIC_USER.Checked
.IsManual = CH_MANUAL.Checked
.Timer = AConvert(Of Integer)(TXT_TIMER.Text, AutoDownloader.DefaultTimer)
.StartupDelay = NUM_DELAY.Value
If .IsManual Then .Stop()
.Update()
End With
MyDefs.CloseForm()

2
SCrawler/Download/Automation/SchedulerEditorForm.vb
View File

@@ -281,7 +281,7 @@ Namespace DownloadObjects
Private Sub BTT_START_FORCE_Click(sender As Object, e As EventArgs) Handles BTT_START_FORCE.Click
If _LatestSelected.ValueBetween(0, LIST_PLANS.Items.Count - 1) Then
With Settings.Automation(_LatestSelected)
If .Working Then .ForceStart() : Refill()
If .Working Or .IsManual Then .ForceStart() : Refill()
End With
End If
End Sub

47
SCrawler/Download/Feed/FeedSpecial.vb
View File

@@ -6,6 +6,7 @@
'
' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY
Imports SCrawler.API.Base
Imports PersonalUtilities.Tools
Imports PersonalUtilities.Functions.XML
Imports UserMediaD = SCrawler.DownloadObjects.TDownloader.UserMediaD
@@ -120,6 +121,52 @@ Namespace DownloadObjects
End If
End Sub
#End Region
#Region "UpdateUsers"
Friend Overloads Sub UpdateUsers(ByVal InitialUser As UserInfo, ByVal NewUser As UserInfo)
Try
If Count > 0 Then
Dim changed As Boolean = False
Dim result As Boolean
Dim item As UserMediaD
For i% = 0 To Count - 1
item = Items(i)
result = False
item = UpdateUsers(item, InitialUser, NewUser, result)
If result Then changed = True : Items(i) = item
Next
If changed Then Save()
End If
Catch ex As Exception
ErrorsDescriber.Execute(EDP.SendToLog, ex, "[FeedSpecial.UpdateUsers]")
MainFrameObj.UpdateLogButton()
End Try
End Sub
Friend Overloads Shared Function UpdateUsers(ByVal Item As UserMediaD, ByVal InitialUser As UserInfo, ByVal NewUser As UserInfo,
ByRef Result As Boolean) As UserMediaD
Dim data As UserMedia
Dim user As IUserData
Dim path$ = InitialUser.File.CutPath.PathWithSeparator
Dim pathNew$ = NewUser.File.CutPath.PathWithSeparator
If Item.UserInfo.Equals(InitialUser) Or Item.UserInfo.Equals(NewUser) Then
If Item.Data.File.PathWithSeparator.Contains(path) Then
data = Item.Data
data.File = data.File.ToString.Replace(path, pathNew)
If Item.User Is Nothing Then
user = Settings.GetUser(NewUser)
Else
user = Item.User
End If
If Not If(user?.IsSubscription, False) Then
Item = New UserMediaD(data, user, Item.Session, Item.Date)
Result = True
Return Item
End If
End If
End If
Result = False
Return Item
End Function
#End Region
#Region "Add"
Friend Overloads Function Add(ByVal Item As UserMediaD, Optional ByVal AutoSave As Boolean = True) As Boolean
If Not Items.Contains(Item) Then

20
SCrawler/Download/Feed/FeedSpecialCollection.vb
View File

@@ -85,11 +85,14 @@ Namespace DownloadObjects
Friend ReadOnly Property Comparer As New FeedSpecial.SEComparer
Private ReadOnly Property ComparerFeeds As New FeedsComparer
Friend ReadOnly Property FeedSpecialRemover As Predicate(Of SFile) = Function(f) f.Name.StartsWith(FeedSpecial.FavoriteName) Or f.Name.StartsWith(FeedSpecial.SpecialName)
''' <summary>InitialUser, NewUser</summary>
Friend ReadOnly Property PendingUsersToUpdate As List(Of KeyValuePair(Of UserInfo, UserInfo))
#End Region
#Region "Initializer, loader, feeds handlers"
Friend Sub New()
FeedAddedEventHandlers = New List(Of FeedActionEventHandler)
FeedRemovedEventHandlers = New List(Of FeedActionEventHandler)
PendingUsersToUpdate = New List(Of KeyValuePair(Of UserInfo, UserInfo))
Feeds = New List(Of FeedSpecial)
End Sub
Friend Sub Load()
@@ -205,6 +208,23 @@ Namespace DownloadObjects
Return result
End Function
#End Region
#Region "UpdateUsers"
Friend Sub UpdateUsers(ByVal InitialUser As UserInfo, ByVal NewUser As UserInfo)
Try
Load()
If Count > 0 Then
Feeds.ForEach(Sub(f) f.UpdateUsers(InitialUser, NewUser))
If Downloader.Files.Count > 0 Then
PendingUsersToUpdate.Add(New KeyValuePair(Of UserInfo, UserInfo)(InitialUser, NewUser))
If Not Downloader.Working Then Downloader.FilesUpdatePendingUsers()
End If
End If
Catch ex As Exception
ErrorsDescriber.Execute(EDP.SendToLog, ex, "[FeedSpecialCollection.UpdateUsers]")
MainFrameObj.UpdateLogButton()
End Try
End Sub
#End Region
#Region "IEnumerable Support"
Private Function GetEnumerator() As IEnumerator(Of FeedSpecial) Implements IEnumerable(Of FeedSpecial).GetEnumerator
Return New MyEnumerator(Of FeedSpecial)(Me)

17
SCrawler/Download/STDownloader/DownloaderUrlsArrForm.Designer.vb generated
View File

@@ -34,7 +34,7 @@ Namespace DownloadObjects.STDownloader
Dim FRM_URLS As System.Windows.Forms.GroupBox
Dim ActionButton5 As PersonalUtilities.Forms.Controls.Base.ActionButton = New PersonalUtilities.Forms.Controls.Base.ActionButton()
Me.TXT_OUTPUT = New PersonalUtilities.Forms.Controls.ComboBoxExtended()
Me.TXT_URLS = New System.Windows.Forms.RichTextBox()
Me.TXT_URLS = New System.Windows.Forms.TextBox()
Me.CMB_ACCOUNT = New PersonalUtilities.Forms.Controls.ComboBoxExtended()
CONTAINER_MAIN = New System.Windows.Forms.ToolStripContainer()
TP_MAIN = New System.Windows.Forms.TableLayoutPanel()
@@ -53,7 +53,7 @@ Namespace DownloadObjects.STDownloader
'CONTAINER_MAIN.ContentPanel
'
CONTAINER_MAIN.ContentPanel.Controls.Add(TP_MAIN)
CONTAINER_MAIN.ContentPanel.Size = New System.Drawing.Size(384, 236)
CONTAINER_MAIN.ContentPanel.Size = New System.Drawing.Size(384, 261)
CONTAINER_MAIN.Dock = System.Windows.Forms.DockStyle.Fill
CONTAINER_MAIN.LeftToolStripPanelVisible = False
CONTAINER_MAIN.Location = New System.Drawing.Point(0, 0)
@@ -77,7 +77,7 @@ Namespace DownloadObjects.STDownloader
TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Absolute, 28.0!))
TP_MAIN.RowStyles.Add(New System.Windows.Forms.RowStyle(System.Windows.Forms.SizeType.Percent, 100.0!))
TP_MAIN.Size = New System.Drawing.Size(384, 236)
TP_MAIN.Size = New System.Drawing.Size(384, 261)
TP_MAIN.TabIndex = 0
'
'TXT_OUTPUT
@@ -126,20 +126,20 @@ Namespace DownloadObjects.STDownloader
FRM_URLS.Dock = System.Windows.Forms.DockStyle.Fill
FRM_URLS.Location = New System.Drawing.Point(3, 59)
FRM_URLS.Name = "FRM_URLS"
FRM_URLS.Size = New System.Drawing.Size(378, 174)
FRM_URLS.Size = New System.Drawing.Size(378, 199)
FRM_URLS.TabIndex = 2
FRM_URLS.TabStop = False
FRM_URLS.Text = "URLs (new line as delimiter)"
'
'TXT_URLS
'
Me.TXT_URLS.DetectUrls = False
Me.TXT_URLS.Dock = System.Windows.Forms.DockStyle.Fill
Me.TXT_URLS.Location = New System.Drawing.Point(3, 16)
Me.TXT_URLS.MaxLength = 2147483647
Me.TXT_URLS.Multiline = True
Me.TXT_URLS.Name = "TXT_URLS"
Me.TXT_URLS.Size = New System.Drawing.Size(372, 155)
Me.TXT_URLS.Size = New System.Drawing.Size(372, 180)
Me.TXT_URLS.TabIndex = 0
Me.TXT_URLS.Text = ""
'
'CMB_ACCOUNT
'
@@ -178,12 +178,13 @@ Namespace DownloadObjects.STDownloader
TP_MAIN.ResumeLayout(False)
CType(Me.TXT_OUTPUT, System.ComponentModel.ISupportInitialize).EndInit()
FRM_URLS.ResumeLayout(False)
FRM_URLS.PerformLayout()
CType(Me.CMB_ACCOUNT, System.ComponentModel.ISupportInitialize).EndInit()
Me.ResumeLayout(False)
End Sub
Private WithEvents TXT_OUTPUT As PersonalUtilities.Forms.Controls.ComboBoxExtended
Private WithEvents TXT_URLS As RichTextBox
Private WithEvents TXT_URLS As TextBox
Private WithEvents CMB_ACCOUNT As PersonalUtilities.Forms.Controls.ComboBoxExtended
End Class
End Namespace

58
SCrawler/Download/TDownloader.vb
View File

@@ -37,19 +37,28 @@ Namespace DownloadObjects
#End Region
Friend ReadOnly User As IUserData
Friend ReadOnly Data As UserMedia
Friend ReadOnly UserInfo As UserInfo
Friend ReadOnly [Date] As Date
Friend ReadOnly Session As Integer
Friend Sub New(ByVal Data As UserMedia, ByVal User As IUserData, ByVal Session As Integer)
Me.Data = Data
Me.User = User
If Not Me.User Is Nothing Then Me.UserInfo = DirectCast(Me.User, UserDataBase).User
[Date] = Now
Me.Session = Session
End Sub
Friend Sub New(ByVal Data As UserMedia, ByVal User As IUserData, ByVal Session As Integer, ByVal _Date As Date)
Me.New(Data, User, Session)
[Date] = _Date
End Sub
Private Sub New(ByVal e As EContainer)
If Not e Is Nothing Then
If e.Contains(Name_User) Then
Dim u As UserInfo = e(Name_User)
If Not u.Name.IsEmptyString And Not u.Site.IsEmptyString Then User = Settings.GetUser(u)
If Not u.Name.IsEmptyString And Not u.Site.IsEmptyString Then
User = Settings.GetUser(u)
If Not User Is Nothing Then UserInfo = DirectCast(User, UserDataBase).User
End If
End If
Data = New UserMedia(e(Name_Media), User)
[Date] = AConvert(Of Date)(e.Value(Name_Date), DateTimeDefaultProvider, Now)
@@ -98,7 +107,7 @@ Namespace DownloadObjects
Return _FilesSessionActual
End Get
End Property
Private Sub FilesSave()
Friend Sub FilesSave()
Try
If Settings.FeedStoreSessionsData And Files.Count > 0 Then
ClearSessions()
@@ -111,6 +120,46 @@ Namespace DownloadObjects
ErrorsDescriber.Execute(EDP.SendToLog, ex, "[DownloadObjects.TDownloader.FilesSave]")
End Try
End Sub
Private _FilesUpdating As Boolean = False
Friend Sub FilesUpdatePendingUsers()
_FilesUpdating = True
Try
If Files.Count > 0 Then
With Settings.Feeds
Dim pUsers As List(Of KeyValuePair(Of UserInfo, UserInfo))
Dim pendingUser As KeyValuePair(Of UserInfo, UserInfo)
Dim item As UserMediaD
Dim result As Boolean
Dim changed As Boolean = False
While .PendingUsersToUpdate.Count > 0
pUsers = New List(Of KeyValuePair(Of UserInfo, UserInfo))(.PendingUsersToUpdate)
.PendingUsersToUpdate.Clear()
For Each pendingUser In pUsers
With Files
If .Count > 0 Then
For i% = 0 To .Count - 1
item = .Item(i)
result = False
item = FeedSpecial.UpdateUsers(item, pendingUser.Key, pendingUser.Value, result)
If result Then changed = True : .Item(i) = item
Next
End If
End With
If changed Then FilesSave()
Next
pUsers.Clear()
End While
End With
End If
Catch aex As ArgumentOutOfRangeException
Catch iex As IndexOutOfRangeException
Catch ex As Exception
ErrorsDescriber.Execute(EDP.SendToLog, ex, "[TDownloader.FilesUpdatePendingUsers]")
MainFrameObj.UpdateLogButton()
Finally
_FilesUpdating = False
End Try
End Sub
Friend Sub ClearSessions()
Try
If Not _FilesSessionCleared Then
@@ -159,7 +208,7 @@ Namespace DownloadObjects
Friend ReadOnly Property Working(Optional ByVal CheckThread As Boolean = True) As Boolean
Get
Return _PoolReconfiguration Or (Pool.Count > 0 AndAlso Pool.Exists(Function(j) j.Working)) Or
(CheckThread AndAlso If(CheckerThread?.IsAlive, False))
(CheckThread AndAlso If(CheckerThread?.IsAlive, False)) Or _FilesUpdating
End Get
End Property
Friend ReadOnly Property Count As Integer
@@ -411,6 +460,7 @@ Namespace DownloadObjects
Files.Sort()
FilesChanged = Not fBefore = Files.Count
RaiseEvent Downloading(False)
FilesUpdatePendingUsers()
If FilesChanged Then FilesSave() : RaiseEvent FeedFilesChanged(True)
End Try
End Sub
@@ -453,7 +503,7 @@ Namespace DownloadObjects
Private Sub UpdateJobsLabel()
RaiseEvent JobsChange(Count)
End Sub
Friend Structure HostLimit
Private Structure HostLimit
Friend Key As String
Friend Limit As Integer
Friend Value As Integer

2
SCrawler/Editors/UserCreatorForm.vb
View File

@@ -402,6 +402,7 @@ Namespace Editors
Else
COLOR_USER.ColorsGetUser(_UserBackColor, _UserForeColor)
Dim tmpUser As UserInfo = User
Dim userBefore As UserInfo = User
Dim accOld$ = tmpUser.AccountName
With tmpUser
.Name = TXT_USER.Text
@@ -462,6 +463,7 @@ Namespace Editors
.ScriptUse = TXT_SCRIPT.Checked
.ScriptData = TXT_SCRIPT.Text
.UpdateUserInformation()
Settings.Feeds.UpdateUsers(userBefore, User)
End With
End If
GoTo CloseForm

4
SCrawler/PluginsEnvironment/Hosts/SettingsHostCollection.vb
View File

@@ -227,15 +227,17 @@ Namespace Plugin.Hosts
End Select
If selectedAcc >= 0 Then
Dim tUser As UserInfo
Dim tUser As UserInfo, userBefore As UserInfo
Dim tUserIndx%
Dim tUserBase As UserDataBase
For Each tUser In users
userBefore = tUser
UpdateUserAccount(tUser, Obj, Hosts(selectedAcc), True, tUserIndx)
tUserBase = .GetUser(tUser)
tUserBase.AccountName = String.Empty
tUserBase.User = tUser
tUserBase.UpdateUserInformation()
Settings.Feeds.UpdateUsers(userBefore, tUser)
changedUsers.Add(tUserBase.ToStringForLog)
Next
.UpdateUsersList()