1.0.1.0

Extend settings, fix minor bugs, add some functions

SCrawler/API/Base/SiteSettings.vb

@@ -60,7 +60,10 @@ Namespace API.Base
                 End If
                 Responser.SaveSettings()
             End If
-            _Path = New XMLValue(Of SFile)("Path", SFile.GetPath($"{GlobalPath.PathWithSeparator}{Site}"), _XML, {Site.ToString}, XMLValue(Of SFile).ToFilePath)
+            _Path = New XMLValue(Of SFile)("Path", SFile.GetPath($"{GlobalPath.PathWithSeparator}{Site}"),
+                                           _XML, {SettingsCLS.Name_Node_Sites, Site.ToString}, XMLValue(Of SFile).ToFilePath)
+            _Path.ReplaceByValue("Path", {Site.ToString})
+            _XML.Remove(Site.ToString)
         End Sub
         Friend Sub Update()
             Responser.SaveSettings()

SCrawler/API/Base/UserDataBase.vb

@@ -13,9 +13,9 @@ Imports UState = SCrawler.API.Base.UserMedia.States
 Namespace API.Base
     Friend MustInherit Class UserDataBase : Implements IUserData
         Friend Const UserFileAppender As String = "User"
-        Friend Event OnPictureUpdated As IUserData.OnPictureUpdatedEventHandler Implements IUserData.OnPictureUpdated
-        Protected Sub Raise_OnPictureUpdated()
-            RaiseEvent OnPictureUpdated(Me)
+        Friend Event OnUserUpdated As IUserData.OnUserUpdatedEventHandler Implements IUserData.OnUserUpdated
+        Protected Sub Raise_OnUserUpdated()
+            RaiseEvent OnUserUpdated(Me)
         End Sub
 #Region "Collection buttons"
         Friend WithEvents BTT_CONTEXT_DOWN As ToolStripMenuItem
@@ -43,6 +43,8 @@ Namespace API.Base
         Private Const Name_Site As String = "Site"
         Private Const Name_IsChannel As String = "IsChannel"
         Private Const Name_UserName As String = "UserName"
+        Private Const Name_UserExists As String = "UserExists"
+        Private Const Name_UserSuspended As String = "UserSuspended"
         Private Const Name_FriendlyName As String = "FriendlyName"
         Private Const Name_UserID As String = "UserID"
         Private Const Name_Description As String = "Description"
@@ -76,6 +78,10 @@ Namespace API.Base
 #Region "Declarations"
         Friend MustOverride Property Site As Sites Implements IContentProvider.Site
         Friend User As UserInfo
+        Protected Const NonExistendUserHelp As String = "404"
+        Protected Const SuspendedUserHelp As String = "403"
+        Friend Overridable Property UserExists As Boolean = True Implements IUserData.Exists
+        Friend Overridable Property UserSuspended As Boolean = False Implements IUserData.Suspended
         Friend Overridable Property Name As String Implements IContentProvider.Name
             Get
                 Return User.Name
@@ -436,10 +442,11 @@ BlockNullPicture:
                 If MyFile.Exists Then
                     FileExists = True
                     Using x As New XmlFile(MyFile) With {.XmlReadOnly = True}
-                        x.DefaultsLoading(False)
                         User.Site = Site
                         Site = x.Value(Name_Site).FromXML(Of Integer)(0)
                         User.Name = x.Value(Name_UserName)
+                        UserExists = x.Value(Name_UserExists).FromXML(Of Boolean)(True)
+                        UserSuspended = x.Value(Name_UserSuspended).FromXML(Of Boolean)(False)
                         ID = x.Value(Name_UserID)
                         FriendlyName = x.Value(Name_FriendlyName)
                         UserDescription = x.Value(Name_Description)
@@ -470,6 +477,8 @@ BlockNullPicture:
                 Using x As New XmlFile With {.Name = "User"}
                     x.Add(Name_Site, CInt(Site))
                     x.Add(Name_UserName, User.Name)
+                    x.Add(Name_UserExists, UserExists.BoolToInteger)
+                    x.Add(Name_UserSuspended, UserSuspended.BoolToInteger)
                     x.Add(Name_UserID, ID)
                     x.Add(Name_FriendlyName, FriendlyName)
                     x.Add(Name_Description, UserDescription)
@@ -508,9 +517,8 @@ BlockNullPicture:
         Private Overloads Sub LoadContentInformation(ByRef _CLIST As List(Of UserMedia), ByVal f As SFile)
             Try
                 If Not f.Exists Then Exit Sub
-                Using x As New XmlFile(f, ProtectionLevels.All, False) With {.XmlReadOnly = True, .AllowSameNames = True}
+                Using x As New XmlFile(f, Protector.Modes.All, False) With {.XmlReadOnly = True, .AllowSameNames = True}
                     x.LoadData()
-                    x.DefaultsLoading(False)
                     If x.Count > 0 Then
                         Dim fs$ = MyFile.CutPath.PathWithSeparator
                         Dim gfn As Func(Of String, String) = Function(ByVal Input As String) As String
@@ -589,11 +597,17 @@ BlockNullPicture:
 #End Region
 #Region "Download functions and options"
         Friend Overridable Property DownloadTopCount As Integer? = Nothing Implements IUserData.DownloadTopCount
+        Protected Responser As PersonalUtilities.Tools.WEB.Response
         Friend Overridable Sub DownloadData(ByVal Token As CancellationToken) Implements IContentProvider.DownloadData
             Dim Canceled As Boolean = False
             Try
                 UpdateDataFiles()
+                If Not Responser Is Nothing Then Responser.Dispose()
+                Responser = New PersonalUtilities.Tools.WEB.Response
+                Responser.Copy(Settings.Site(Site).Responser)
                 Dim UpPic As Boolean = Settings.ViewModeIsPicture AndAlso GetPicture(False) Is Nothing
+                Dim sEnvir() As Boolean = {UserExists, UserSuspended}
+                Dim EnvirChanged As Func(Of Boolean) = Function() Not sEnvir(0) = UserExists Or Not sEnvir(1) = UserSuspended
                 _DownloadedPicturesSession = 0
                 _DownloadedVideosSession = 0
                 _TempMediaList.Clear()
@@ -623,7 +637,7 @@ BlockNullPicture:
                 _ContentList.ListAddList(_ContentNew.Where(Function(c) c.State = UState.Downloaded), LNC)
                 _CountPictures = _ContentList.LongCount(Function(c) c.Type = UserMedia.Types.Picture)
                 _CountVideo = _ContentList.LongCount(Function(c) c.Type = UserMedia.Types.Video)
-                If DownloadedPictures + DownloadedVideos > 0 Then
+                If DownloadedPictures + DownloadedVideos > 0 Or EnvirChanged.Invoke Then
                     If __SaveData Then
                         LastUpdated = Now
                         If Labels.Contains(LabelsKeeper.NoParsedUser) Then Labels.Remove(LabelsKeeper.NoParsedUser)
@@ -634,12 +648,13 @@ BlockNullPicture:
                         _ContentList.Clear()
                         CreatedByChannel = False
                     End If
+                    If Not UserExists Then ReadyForDownload = False
                     UpdateUserInformation()
                 End If
                 ThrowIfDisposed()
                 _DownloadedPicturesTotal += _DownloadedPicturesSession
                 _DownloadedVideosTotal += _DownloadedVideosSession
-                If UpPic Then Raise_OnPictureUpdated()
+                If UpPic Or EnvirChanged.Invoke Then Raise_OnUserUpdated()
             Catch oex As OperationCanceledException When Token.IsCancellationRequested
                 MyMainLOG = $"{Site} - {Name}: downloading canceled"
                 Canceled = True
@@ -649,6 +664,7 @@ BlockNullPicture:
                 LogError(ex, "downloading data error")
                 HasError = True
             Finally
+                If Not Responser Is Nothing Then Responser.Dispose() : Responser = Nothing
                 If Not Canceled Then _DataParsed = True ': LastUpdated = Now
                 _ContentNew.Clear()
                 DownloadTopCount = Nothing
@@ -927,6 +943,7 @@ BlockNullPicture:
                     _ContentNew.Clear()
                     _TempMediaList.Clear()
                     _TempPostsList.Clear()
+                    If Not Responser Is Nothing Then Responser.Dispose()
                     If Not BTT_CONTEXT_DOWN Is Nothing Then BTT_CONTEXT_DOWN.Dispose()
                     If Not BTT_CONTEXT_EDIT Is Nothing Then BTT_CONTEXT_EDIT.Dispose()
                     If Not BTT_CONTEXT_DELETE Is Nothing Then BTT_CONTEXT_DELETE.Dispose()
@@ -959,7 +976,7 @@ BlockNullPicture:
         Sub DownloadData(ByVal Token As CancellationToken)
     End Interface
     Friend Interface IUserData : Inherits IContentProvider, IComparable(Of UserDataBase), IComparable, IEquatable(Of UserDataBase), IDisposable
-        Event OnPictureUpdated(ByVal User As IUserData)
+        Event OnUserUpdated(ByVal User As IUserData)
         Property ParseUserMediaOnly As Boolean
 #Region "Images"
         Function GetPicture() As Image
@@ -972,6 +989,8 @@ BlockNullPicture:
         ReadOnly Property Labels As List(Of String)
 #End Region
         ReadOnly Property IsChannel As Boolean
+        Property Exists As Boolean
+        Property Suspended As Boolean
         Property ReadyForDownload As Boolean
         Property [File] As SFile
         Property FileExists As Boolean

SCrawler/API/Reddit/Channel.vb

@@ -22,9 +22,23 @@ Namespace API.Reddit
 #End Region
         Friend Const DefaultDownloadLimitCount As Integer = 1000
 #Region "IUserData Support"
-        Private Event OnPictureUpdated(User As IUserData) Implements IUserData.OnPictureUpdated
+        Private Event OnUserUpdated As IUserData.OnUserUpdatedEventHandler Implements IUserData.OnUserUpdated
         Friend Property Instance As IUserData
         Private Property IUserData_ParseUserMediaOnly As Boolean = False Implements IUserData.ParseUserMediaOnly
+        Private Property IUserData_Exists As Boolean Implements IUserData.Exists
+            Get
+                Return Instance.Exists
+            End Get
+            Set(ByVal e As Boolean)
+            End Set
+        End Property
+        Private Property IUserData_Suspended As Boolean Implements IUserData.Suspended
+            Get
+                Return Instance.Suspended
+            End Get
+            Set(ByVal s As Boolean)
+            End Set
+        End Property
         Private ReadOnly Property IUserData_IsCollection As Boolean Implements IUserData.IsCollection
             Get
                 Return Instance.IsCollection
@@ -504,9 +518,8 @@ Namespace API.Reddit
         End Function
         Friend Overloads Function LoadData(ByVal f As SFile, ByVal PartialLoad As Boolean, Optional ByVal e As ErrorsDescriber = Nothing) As Boolean
             If f.Exists Then
-                Using x As New XmlFile(f, ProtectionLevels.All, False) With {.XmlReadOnly = True, .AllowSameNames = True}
+                Using x As New XmlFile(f, Protector.Modes.All, False) With {.XmlReadOnly = True, .AllowSameNames = True}
                     x.LoadData()
-                    x.DefaultsLoading(False)
                     If x.Count > 0 Then
                         Dim XMLDateProvider As New ADateTime(ADateTime.Formats.BaseDateTime)
                         Name = x.Value(Name_Name)
@@ -527,7 +540,6 @@ Namespace API.Reddit
         Friend Overloads Function Save(Optional ByVal f As SFile = Nothing, Optional ByVal e As ErrorsDescriber = Nothing) As Boolean Implements ILoaderSaver.Save
             Dim XMLDateProvider As New ADateTime(ADateTime.Formats.BaseDateTime)
             Using x As New XmlFile With {.AllowSameNames = True, .Name = "Channel"}
-                x.DefaultsLoading(False)
                 x.Add(Name_Name, Name)
                 x.Add(Name_ID, ID)
                 If Posts.Count > 0 Or PostsLatest.Count > 0 Then

SCrawler/API/Reddit/UserData.vb

@@ -62,6 +62,8 @@ Namespace API.Reddit
 #Region "Initializers"
         ''' <summary>Video downloader initializer</summary>
         Private Sub New()
+            ChannelPostsNames = New List(Of String)
+            _ExistsUsersNames = New List(Of String)
         End Sub
         ''' <summary>Default initializer</summary>
         Friend Sub New(ByVal u As UserInfo, Optional ByVal _LoadUserInformation As Boolean = True, Optional ByVal InvokeImageHandler As Boolean = True)
@@ -75,6 +77,9 @@ Namespace API.Reddit
 #Region "Download Overrides"
         Friend Overrides Sub DownloadData(ByVal Token As CancellationToken)
             If IsChannel AndAlso Not ChannelInfo.IsRegularChannel Then
+                If Not Responser Is Nothing Then Responser.Dispose()
+                Responser = New PersonalUtilities.Tools.WEB.Response
+                Responser.Copy(Settings.Site(Sites.Reddit).Responser)
                 ChannelPostsNames.ListAddList(ChannelInfo.PostsAll.Select(Function(p) p.ID), LNC)
                 If SkipExistsUsers Then _ExistsUsersNames.ListAddList(Settings.UsersList.Select(Function(p) p.Name), LNC)
                 DownloadDataF(Token)
@@ -201,8 +206,14 @@ Namespace API.Reddit
             Catch oex As OperationCanceledException When Token.IsCancellationRequested
             Catch dex As ObjectDisposedException When Disposed
             Catch ex As Exception
-                LogError(ex, $"data downloading error [{URL}]")
-                HasError = True
+                If ex.HelpLink = NonExistendUserHelp Then
+                    UserExists = False
+                ElseIf ex.HelpLink = SuspendedUserHelp Then
+                    UserSuspended = True
+                Else
+                    LogError(ex, $"data downloading error [{URL}]")
+                    HasError = True
+                End If
             End Try
         End Sub
         Private Sub DownloadDataChannel(ByVal POST As String, ByVal Token As CancellationToken)
@@ -283,8 +294,14 @@ Namespace API.Reddit
             Catch oex As OperationCanceledException When Token.IsCancellationRequested
             Catch dex As ObjectDisposedException When Disposed
             Catch ex As Exception
-                LogError(ex, $"channel data downloading error [{URL}]")
-                HasError = True
+                If ex.HelpLink = NonExistendUserHelp Then
+                    UserExists = False
+                ElseIf ex.HelpLink = SuspendedUserHelp Then
+                    UserSuspended = True
+                Else
+                    LogError(ex, $"channel data downloading error [{URL}]")
+                    HasError = True
+                End If
             End Try
         End Sub
 #End Region
@@ -384,6 +401,8 @@ Namespace API.Reddit
                 If Not URL.IsEmptyString AndAlso URL.Contains("redgifs") Then
                     Using r As New UserData
                         r._TempMediaList.Add(MediaFromData(UTypes.VideoPre, URL, String.Empty, String.Empty,, False))
+                        r.Responser = New PersonalUtilities.Tools.WEB.Response
+                        r.Responser.Copy(Settings.Site(Sites.Reddit).Responser)
                         r.ReparseVideo(Nothing)
                         If r._TempMediaList.ListExists Then Return r._TempMediaList(0)
                     End Using
@@ -442,6 +461,7 @@ Namespace API.Reddit
                         Dim cached As Boolean = IsChannel And SaveToCache
                         Dim vsf As Boolean = SeparateVideoFolderF
                         Dim ImgFormat As Imaging.ImageFormat
+                        Dim UseMD5 As Boolean = Not IsChannel Or (Not cached And Settings.ChannelsRegularCheckMD5)
                         Dim bDP As New ErrorsDescriber(EDP.None)
                         Dim MD5BS As Func(Of String, UTypes,
                                              SFile, Boolean, String) = Function(ByVal __URL As String, ByVal __MT As UTypes,
@@ -474,7 +494,7 @@ Namespace API.Reddit
                                 End If
                                 f.Separator = "\"
                                 m = String.Empty
-                                If (v.Type = UTypes.Picture Or v.Type = UTypes.GIF) And Not cached Then
+                                If (v.Type = UTypes.Picture Or v.Type = UTypes.GIF) And UseMD5 Then
                                     m = MD5BS(v.URL, v.Type, f, False)
                                     If m.IsEmptyString AndAlso Not v.URL_BASE.IsEmptyString AndAlso Not v.URL_BASE = v.URL Then
                                         m = MD5BS(v.URL_BASE, v.Type, f, True)
@@ -483,7 +503,7 @@ Namespace API.Reddit
                                 End If
 
                                 If (Not m.IsEmptyString AndAlso Not HashList.Contains(m)) Or Not (v.Type = UTypes.Picture Or
-                                                                                                  v.Type = UTypes.GIF) Or cached Then
+                                                                                                  v.Type = UTypes.GIF) Or Not UseMD5 Then
                                     If Not cached Then HashList.Add(m)
                                     v.MD5 = m
                                     f.Path = MyDir
@@ -536,15 +556,20 @@ Namespace API.Reddit
         End Sub
         Protected Function GetSiteResponse(ByVal URL As String, Optional ByVal e As ErrorsDescriber = Nothing) As String
             Try
-                Return Settings.Site(Sites.Reddit).Responser.GetResponse(URL,, EDP.ThrowException)
+                Return Responser.GetResponse(URL,, EDP.ThrowException)
             Catch ex As Exception
                 HasError = True
                 Dim OptText$ = String.Empty
                 If Not e.Exists Then
                     Dim ee As EDP = EDP.SendInLog
-                    If Settings.Site(Sites.Reddit).Responser.StatusCode = HttpStatusCode.NotFound Then
-                        ee += EDP.ThrowException
+                    If Responser.StatusCode = HttpStatusCode.NotFound Then
+                        ee = EDP.ThrowException
                         OptText = ": USER NOT FOUND"
+                        ex.HelpLink = NonExistendUserHelp
+                    ElseIf Responser.StatusCode = HttpStatusCode.Forbidden Then
+                        ee = EDP.ThrowException
+                        OptText = ": USER PROFILE SUSPENDED"
+                        ex.HelpLink = SuspendedUserHelp
                     Else
                         ee += EDP.ReturnValue
                     End If

SCrawler/API/Twitter/UserData.vb

@@ -6,6 +6,7 @@
 '
 ' This program is distributed in the hope that it will be useful,
 ' but WITHOUT ANY WARRANTY
+Imports PersonalUtilities.Tools.WEB
 Imports PersonalUtilities.Tools.WebDocuments.JSON
 Imports PersonalUtilities.Functions.XML
 Imports System.Net
@@ -67,7 +68,7 @@ Namespace API.Twitter
                 If Not POST.IsEmptyString Then URL &= $"&max_id={POST}"
 
                 ThrowAny(Token)
-                Dim r$ = Settings.Site(Sites.Twitter).Responser.GetResponse(URL,, EDP.ThrowException)
+                Dim r$ = Responser.GetResponse(URL,, EDP.ThrowException)
                 If Not r.IsEmptyString Then
                     Using w As EContainer = JsonDocument.Parse(r)
                         If Not w Is Nothing AndAlso w.Count > 0 Then
@@ -117,8 +118,16 @@ Namespace API.Twitter
             Catch oex As OperationCanceledException When Token.IsCancellationRequested
             Catch dex As ObjectDisposedException When Disposed
             Catch ex As Exception
-                LogError(ex, $"data downloading error [{URL}]")
-                HasError = True
+                If Responser.StatusCode = HttpStatusCode.NotFound Then
+                    UserExists = False
+                ElseIf Responser.StatusCode = HttpStatusCode.Unauthorized Then
+                    UserSuspended = True
+                ElseIf Responser.StatusCode = HttpStatusCode.BadRequest Then
+                    MyMainLOG = "Twitter has invalid credentials"
+                Else
+                    LogError(ex, $"data downloading error [{URL}]")
+                    HasError = True
+                End If
             End Try
         End Sub
         Friend Shared Function GetVideoInfo(ByVal URL As String) As UserMedia
@@ -126,8 +135,8 @@ Namespace API.Twitter
                 If URL.Contains("twitter") Then
                     Dim PostID$ = RegexReplace(URL, New RegexStructure("(?<=/)\d+", True, False,,,,, String.Empty))
                     If Not PostID.IsEmptyString Then
-                        Dim r$ = Settings.Site(Sites.Twitter).Responser.GetResponse($"https://api.twitter.com/1.1/statuses/show.json?id={PostID}",,
-                                                                                    EDP.ReturnValue)
+                        Dim r$ = DirectCast(Settings.Site(Sites.Twitter).Responser.Copy(), Response).
+                                            GetResponse($"https://api.twitter.com/1.1/statuses/show.json?id={PostID}",, EDP.ReturnValue)
                         If Not r.IsEmptyString Then
                             Using j As EContainer = JsonDocument.Parse(r)
                                 If j.ListExists Then

SCrawler/API/UserDataBind.vb

@@ -36,6 +36,20 @@ Namespace API
                 CollectionName = NewCollectionName
             End Set
         End Property
+        Friend Overrides Property UserExists As Boolean
+            Get
+                Return Count > 0 AndAlso Collections.Exists(Function(c) c.Exists)
+            End Get
+            Set(ByVal e As Boolean)
+            End Set
+        End Property
+        Friend Overrides Property UserSuspended As Boolean
+            Get
+                Return Count > 0 AndAlso Collections.LongCount(Function(c) c.Suspended) = Count
+            End Get
+            Set(ByVal s As Boolean)
+            End Set
+        End Property
         Friend Overrides Sub ChangeCollectionName(ByVal NewName As String, ByVal UpdateSettings As Boolean)
             _CollectionName = NewName
             If Count > 0 Then Collections.ForEach(Sub(c) c.CollectionName = NewName)
@@ -278,8 +292,8 @@ Namespace API
         End Sub
         Protected Overrides Sub DownloadContent(ByVal Token As CancellationToken)
         End Sub
-        Private Sub User_OnPictureUpdated(ByVal User As IUserData)
-            Raise_OnPictureUpdated()
+        Private Sub User_OnUserUpdated(ByVal User As IUserData)
+            Raise_OnUserUpdated()
         End Sub
         Friend Overrides Sub OpenSite()
             If Count > 0 Then Collections(0).OpenSite()
@@ -314,7 +328,7 @@ Namespace API
                             .UpdateUserInformation()
                         End If
                         ImageHandler(_Item, False)
-                        AddHandler .Self.OnPictureUpdated, AddressOf User_OnPictureUpdated
+                        AddHandler .Self.OnUserUpdated, AddressOf User_OnUserUpdated
                         DirectCast(.Self, UserDataBase).CreateButtons(Count - 1)
                     End With
                 Else
@@ -333,7 +347,7 @@ Namespace API
                 .CreateButtons(Count - 1)
                 AddHandler .BTT_CONTEXT_DELETE.Click, AddressOf BTT_CONTEXT_DELETE_Click
             End With
-            AddHandler Collections(Count - 1).OnPictureUpdated, AddressOf User_OnPictureUpdated
+            AddHandler Collections.Last.OnUserUpdated, AddressOf User_OnUserUpdated
         End Sub
         Friend Sub AddRange(ByVal _Items As IEnumerable(Of IUserData))
             If Not _Items Is Nothing AndAlso _Items.Count > 0 Then