geonu/SCrawler
1
0
Fork 0
mirror of https://github.com/AAndyProgram/SCrawler.git synced 2026-03-14 15:52:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

3.0.0.4

Browse Source 
Changed XVIDEOS.M3U8 errors
Added TryCatch to XVIDEOS.UserData.Download
Removed old declarations in UserDataBase
Changed description replacement in UserDataHost
This commit is contained in:
Andy
2022-03-26 20:00:55 +03:00
parent 726fc486ce
commit 975d46715c
9 changed files with 55 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Changelog.md
View File

@@ -1,3 +1,9 @@
# 3.0.0.4
- Fixed
- External plugins do not save information about downloaded files
- The user cannot be added to the collection if a special path has been specified.
# 3.0.0.3 # 3.0.0.3
- Added - Added

4
SCrawler.Plugin.LPSG/My Project/AssemblyInfo.vb
View File

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below: ' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")> ' <Assembly: AssemblyVersion("1.0.*")>
<Assembly: AssemblyVersion("1.0.0.0")> <Assembly: AssemblyVersion("1.0.0.1")>
<Assembly: AssemblyFileVersion("1.0.0.0")> <Assembly: AssemblyFileVersion("1.0.0.1")>
<Assembly: NeutralResourcesLanguage("en")> <Assembly: NeutralResourcesLanguage("en")>

15
SCrawler.Plugin.LPSG/UserData.vb
View File

@@ -7,6 +7,8 @@
' This program is distributed in the hope that it will be useful, ' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY ' but WITHOUT ANY WARRANTY
Imports PersonalUtilities.Functions.RegularExpressions Imports PersonalUtilities.Functions.RegularExpressions
Imports UStates = SCrawler.Plugin.PluginUserMedia.States
Imports UTypes = SCrawler.Plugin.PluginUserMedia.Types
Public Class UserData : Implements IPluginContentProvider Public Class UserData : Implements IPluginContentProvider
#Region "XML names" #Region "XML names"
Private Const Name_LatestPage As String = "LatestPage" Private Const Name_LatestPage As String = "LatestPage"
@@ -65,7 +67,7 @@ Public Class UserData : Implements IPluginContentProvider
With Responser : .Copy(Settings.Responser) : .Error = EDP.ThrowException : End With With Responser : .Copy(Settings.Responser) : .Error = EDP.ThrowException : End With
Dim l As List(Of String) = Nothing Dim l As List(Of String) = Nothing
Dim NextPage$ = String.Empty Dim NextPage$
Dim r$ Dim r$
Dim _LPage As Func(Of String) = Function() If(LatestPage.IsEmptyString, String.Empty, $"page-{LatestPage}") Dim _LPage As Func(Of String) = Function() If(LatestPage.IsEmptyString, String.Empty, $"page-{LatestPage}")
@@ -90,11 +92,11 @@ Public Class UserData : Implements IPluginContentProvider
f = CStr(RegexReplace(u, FileRegEx)) f = CStr(RegexReplace(u, FileRegEx))
f.Path = DataPath.CSFilePSN f.Path = DataPath.CSFilePSN
f.Separator = "\" f.Separator = "\"
TempMediaList.Add(New PluginUserMedia With {.ContentType = PluginUserMedia.Types.Picture, .URL = u, .File = f}) TempMediaList.Add(New PluginUserMedia With {.ContentType = UTypes.Picture, .URL = u, .File = f})
End If End If
Next Next
If TempMediaList.ListExists And ExistingContentList.ListExists Then _ If TempMediaList.ListExists And ExistingContentList.ListExists Then _
TempMediaList.RemoveAll(Function(m) ExistingContentList.Exists(Function(mm) mm.URL = m.URL)) TempMediaList.RemoveAll(Function(m) ExistingContentList.Exists(Function(mm) mm.URL = m.URL))
End If End If
Catch oex As OperationCanceledException Catch oex As OperationCanceledException
Catch dex As ObjectDisposedException Catch dex As ObjectDisposedException
@@ -112,15 +114,16 @@ Public Class UserData : Implements IPluginContentProvider
For i% = 0 To TempMediaList.Count - 1 For i% = 0 To TempMediaList.Count - 1
Thrower.ThrowAny() Thrower.ThrowAny()
m = TempMediaList(i) m = TempMediaList(i)
m.DownloadState = PluginUserMedia.States.Tried m.DownloadState = UStates.Tried
Try Try
If Not m.URL.IsEmptyString And Not m.File.IsEmptyString Then If Not m.URL.IsEmptyString And Not m.File.IsEmptyString Then
Responser.DownloadFile(m.URL, m.File, eweb) Responser.DownloadFile(m.URL, m.File, eweb)
m.DownloadState = PluginUserMedia.States.Downloaded m.DownloadState = UStates.Downloaded
Else Else
m.DownloadState = PluginUserMedia.States.Skipped m.DownloadState = UStates.Skipped
End If End If
Catch ex As Exception Catch ex As Exception
m.DownloadState = UStates.Skipped
End Try End Try
RaiseEvent ProgressChanged(1) RaiseEvent ProgressChanged(1)
TempMediaList(i) = m TempMediaList(i) = m

7
SCrawler.Plugin.XVIDEOS/M3U8.vb
View File

@@ -41,7 +41,8 @@ Friend NotInheritable Class M3U8
Return Nothing Return Nothing
Catch ex As Exception Catch ex As Exception
Logger.Add(ex, "[M3U8.Save]") Logger.Add(ex, "[M3U8.Save]")
Return Nothing ex.HelpLink = 1
Throw ex
Finally Finally
CachePath.Delete(SFO.Path, SFODelete.None, EDP.None) CachePath.Delete(SFO.Path, SFODelete.None, EDP.None)
End Try End Try
@@ -61,8 +62,8 @@ Friend NotInheritable Class M3U8
End If End If
Return Nothing Return Nothing
Catch ex As Exception Catch ex As Exception
Logger.Add(ex, "[M3U8.Download]") If Not ex.HelpLink = 1 Then Logger.Add(ex, "[M3U8.Download]")
Return Nothing Throw ex
End Try End Try
End Function End Function
End Class End Class

4
SCrawler.Plugin.XVIDEOS/My Project/AssemblyInfo.vb
View File

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below: ' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")> ' <Assembly: AssemblyVersion("1.0.*")>
<Assembly: AssemblyVersion("1.0.0.1")> <Assembly: AssemblyVersion("1.0.0.2")>
<Assembly: AssemblyFileVersion("1.0.0.1")> <Assembly: AssemblyFileVersion("1.0.0.2")>
<Assembly: NeutralResourcesLanguage("en")> <Assembly: NeutralResourcesLanguage("en")>

17
SCrawler.Plugin.XVIDEOS/UserData.vb
View File

@@ -10,6 +10,8 @@ Imports PersonalUtilities.Functions.RegularExpressions
Imports PersonalUtilities.Functions.XML Imports PersonalUtilities.Functions.XML
Imports PersonalUtilities.Tools.WEB Imports PersonalUtilities.Tools.WEB
Imports PersonalUtilities.Tools.WebDocuments.JSON Imports PersonalUtilities.Tools.WebDocuments.JSON
Imports UStates = SCrawler.Plugin.PluginUserMedia.States
Imports UTypes = SCrawler.Plugin.PluginUserMedia.Types
Public Class UserData : Implements IPluginContentProvider Public Class UserData : Implements IPluginContentProvider
#Region "Interface declarations" #Region "Interface declarations"
Public Event ProgressChanged(Count As Integer) Implements IPluginContentProvider.ProgressChanged Public Event ProgressChanged(Count As Integer) Implements IPluginContentProvider.ProgressChanged
@@ -71,7 +73,7 @@ Public Class UserData : Implements IPluginContentProvider
Thrower.ThrowAny() Thrower.ThrowAny()
r = Responser.GetResponse($"https://www.xvideos.com/{user}/videos/new/{If(NextPage = 0, String.Empty, NextPage)}",, e) r = Responser.GetResponse($"https://www.xvideos.com/{user}/videos/new/{If(NextPage = 0, String.Empty, NextPage)}",, e)
If Not r.IsEmptyString Then If Not r.IsEmptyString Then
If Not EnvirSet Then UserExists = True : UserSuspended = False If Not EnvirSet Then UserExists = True : UserSuspended = False : EnvirSet = True
j = JsonDocument.Parse(r).XmlIfNothing j = JsonDocument.Parse(r).XmlIfNothing
With j With j
If .Contains("videos") Then If .Contains("videos") Then
@@ -165,7 +167,7 @@ Public Class UserData : Implements IPluginContentProvider
End If End If
If Not m.IsEmptyString Then If Not m.IsEmptyString Then
Return New PluginUserMedia With { Return New PluginUserMedia With {
.ContentType = PluginUserMedia.Types.m3u8, .ContentType = UTypes.m3u8,
.PostID = pID, .PostID = pID,
.URL = m, .URL = m,
.File = $"{t}.mp4", .File = $"{t}.mp4",
@@ -195,9 +197,14 @@ Public Class UserData : Implements IPluginContentProvider
m = TempMediaList(i) m = TempMediaList(i)
f = m.File f = m.File
f.Path = DefPath f.Path = DefPath
f = M3U8.Download(m.URL, m.SpecialFolder, Settings.FfmpegFile, f, LogProvider) m.DownloadState = UStates.Tried
m.File = f Try
If Not f.IsEmptyString Then m.DownloadState = PluginUserMedia.States.Downloaded f = M3U8.Download(m.URL, m.SpecialFolder, Settings.FfmpegFile, f, LogProvider)
m.File = f
m.DownloadState = UStates.Downloaded
Catch ex As Exception
m.DownloadState = UStates.Skipped
End Try
TempMediaList(i) = m TempMediaList(i) = m
RaiseEvent ProgressChanged(1) RaiseEvent ProgressChanged(1)
Next Next

19
SCrawler/API/Base/UserDataBase.vb
View File

@@ -941,16 +941,13 @@ BlockNullPicture:
#End Region #End Region
#Region "Delete, Move, Merge" #Region "Delete, Move, Merge"
Friend Overridable Function Delete() As Integer Implements IUserData.Delete Friend Overridable Function Delete() As Integer Implements IUserData.Delete
Return DeleteF(Me)
End Function
Friend Function DeleteF(ByVal Instance As IUserData) As Integer
Dim f As SFile = SFile.GetPath(MyFile.CutPath.Path) Dim f As SFile = SFile.GetPath(MyFile.CutPath.Path)
If f.Exists(SFO.Path, False) AndAlso (User.Merged OrElse f.Delete(SFO.Path, Settings.DeleteMode)) Then If f.Exists(SFO.Path, False) AndAlso (User.Merged OrElse f.Delete(SFO.Path, Settings.DeleteMode)) Then
ImageHandler(Me, False) ImageHandler(Me, False)
Settings.UsersList.Remove(User) Settings.UsersList.Remove(User)
Settings.UpdateUsersList() Settings.UpdateUsersList()
Settings.Users.Remove(Instance) Settings.Users.Remove(Me)
Downloader.UserRemove(Instance) Downloader.UserRemove(Me)
Dispose(True) Dispose(True)
Return 1 Return 1
Else Else
@@ -958,24 +955,22 @@ BlockNullPicture:
End If End If
End Function End Function
Friend Overridable Function MoveFiles(ByVal __CollectionName As String) As Boolean Implements IUserData.MoveFiles Friend Overridable Function MoveFiles(ByVal __CollectionName As String) As Boolean Implements IUserData.MoveFiles
Return MoveFilesF(Me, __CollectionName)
End Function
Friend Function MoveFilesF(ByRef Instance As IUserData, ByVal __CollectionName As String) As Boolean
Dim UserBefore As UserInfo = User Dim UserBefore As UserInfo = User
Dim Removed As Boolean = True Dim Removed As Boolean = True
Dim _TurnBack As Boolean = False Dim _TurnBack As Boolean = False
Try Try
Dim f As SFile Dim f As SFile
If IncludedInCollection Then If IncludedInCollection Then
Settings.Users.Add(Instance) Settings.Users.Add(Me)
Removed = False Removed = False
User.CollectionName = String.Empty User.CollectionName = String.Empty
User.IncludedInCollection = False User.IncludedInCollection = False
Else Else
Settings.Users.Remove(Instance) Settings.Users.Remove(Me)
Removed = True Removed = True
User.CollectionName = __CollectionName User.CollectionName = __CollectionName
User.IncludedInCollection = True User.IncludedInCollection = True
User.SpecialPath = Nothing
End If End If
_TurnBack = True _TurnBack = True
User.UpdateUserFile() User.UpdateUserFile()
@@ -987,7 +982,7 @@ BlockNullPicture:
"Destination directory is not empty!"}, MsgBoxStyle.Exclamation,,, {"Delete", "Cancel"}) = 1 Then "Destination directory is not empty!"}, MsgBoxStyle.Exclamation,,, {"Delete", "Cancel"}) = 1 Then
MsgBoxE("Operation canceled", MsgBoxStyle.Exclamation) MsgBoxE("Operation canceled", MsgBoxStyle.Exclamation)
User = UserBefore User = UserBefore
If Removed Then Settings.Users.Add(Instance) Else Settings.Users.Remove(Instance) If Removed Then Settings.Users.Add(Me) Else Settings.Users.Remove(Me)
_TurnBack = False _TurnBack = False
Return False Return False
End If End If
@@ -1003,7 +998,7 @@ BlockNullPicture:
ErrorsDescriber.Execute(EDP.LogMessageValue, ex, "Files moving error") ErrorsDescriber.Execute(EDP.LogMessageValue, ex, "Files moving error")
User = UserBefore User = UserBefore
If _TurnBack Then If _TurnBack Then
If Removed Then Settings.Users.Add(Instance) Else Settings.Users.Remove(Instance) If Removed Then Settings.Users.Add(Me) Else Settings.Users.Remove(Me)
End If End If
Return False Return False
End Try End Try

4
SCrawler/My Project/AssemblyInfo.vb
View File

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
' by using the '*' as shown below: ' by using the '*' as shown below:
' <Assembly: AssemblyVersion("1.0.*")> ' <Assembly: AssemblyVersion("1.0.*")>
<Assembly: AssemblyVersion("3.0.0.3")> <Assembly: AssemblyVersion("3.0.0.4")>
<Assembly: AssemblyFileVersion("3.0.0.3")> <Assembly: AssemblyFileVersion("3.0.0.4")>
<Assembly: NeutralResourcesLanguage("en")> <Assembly: NeutralResourcesLanguage("en")>

14
SCrawler/PluginsEnvironment/Hosts/UserDataHost.vb
View File

@@ -6,10 +6,12 @@
' '
' This program is distributed in the hope that it will be useful, ' This program is distributed in the hope that it will be useful,
' but WITHOUT ANY WARRANTY ' but WITHOUT ANY WARRANTY
Imports SCrawler.API.Base
Imports PersonalUtilities.Functions.XML Imports PersonalUtilities.Functions.XML
Imports SCrawler.API.Base
Imports System.Threading Imports System.Threading
Imports System.Reflection Imports System.Reflection
Imports UStates = SCrawler.Plugin.PluginUserMedia.States
Imports UTypes = SCrawler.Plugin.PluginUserMedia.Types
Namespace Plugin.Hosts Namespace Plugin.Hosts
Friend Class UserDataHost : Inherits UserDataBase Friend Class UserDataHost : Inherits UserDataBase
Private ReadOnly UseInternalDownloader As Boolean Private ReadOnly UseInternalDownloader As Boolean
@@ -67,7 +69,7 @@ Namespace Plugin.Hosts
If Not .Name = Name Then Name = .Name If Not .Name = Name Then Name = .Name
ID = .ID ID = .ID
UserDescription = .UserDescription UserDescriptionUpdate(.UserDescription)
UserExists = .UserExists UserExists = .UserExists
UserSuspended = .UserSuspended UserSuspended = .UserSuspended
End With End With
@@ -84,7 +86,13 @@ Namespace Plugin.Hosts
.TempMediaList.ListAddList(_ContentNew.Select(Function(c) c.PluginUserMedia())) .TempMediaList.ListAddList(_ContentNew.Select(Function(c) c.PluginUserMedia()))
.Download() .Download()
_ContentNew.Clear() _ContentNew.Clear()
If .TempMediaList.ListExists Then _ContentNew.ListAddList(.TempMediaList.Select(Function(c) New UserMedia(c))) If .TempMediaList.ListExists Then
_ContentNew.ListAddList(.TempMediaList.Select(Function(c) New UserMedia(c)))
DownloadedPictures(False) = .TempMediaList.LongCount(Function(m) m.DownloadState = UStates.Downloaded And
(m.ContentType = UTypes.Picture Or m.ContentType = UTypes.GIF))
DownloadedVideos(False) = .TempMediaList.LongCount(Function(m) m.DownloadState = UStates.Downloaded And
(m.ContentType = UTypes.Video Or m.ContentType = UTypes.m3u8))
End If
End With End With
End If End If
End Sub End Sub