' Copyright (C) 2022 Andy ' This program is free software: you can redistribute it and/or modify ' it under the terms of the GNU General Public License as published by ' the Free Software Foundation, either version 3 of the License, or ' (at your option) any later version. ' ' This program is distributed in the hope that it will be useful, ' but WITHOUT ANY WARRANTY Imports PersonalUtilities.Functions.RegularExpressions Imports PersonalUtilities.Functions.XML Imports PersonalUtilities.Tools.WEB Imports PersonalUtilities.Tools.WebDocuments.JSON Imports UStates = SCrawler.Plugin.PluginUserMedia.States Imports UTypes = SCrawler.Plugin.PluginUserMedia.Types Public Class UserData : Implements IPluginContentProvider #Region "Interface declarations" Public Event ProgressChanged(ByVal Count As Integer) Implements IPluginContentProvider.ProgressChanged Public Event TotalCountChanged(ByVal Count As Integer) Implements IPluginContentProvider.TotalCountChanged Public Property Thrower As IThrower Implements IPluginContentProvider.Thrower Public Property LogProvider As ILogProvider Implements IPluginContentProvider.LogProvider Public Property ESettings As ISiteSettings Implements IPluginContentProvider.Settings Private ReadOnly Property Settings As SiteSettings Get Return DirectCast(ESettings, SiteSettings) End Get End Property Public Property Name As String Implements IPluginContentProvider.Name Public Property ID As String Implements IPluginContentProvider.ID Public Property ParseUserMediaOnly As Boolean Implements IPluginContentProvider.ParseUserMediaOnly Public Property UserDescription As String Implements IPluginContentProvider.UserDescription Public Property ExistingContentList As List(Of PluginUserMedia) Implements IPluginContentProvider.ExistingContentList Public Property TempPostsList As List(Of String) Implements IPluginContentProvider.TempPostsList Public Property TempMediaList As List(Of PluginUserMedia) Implements IPluginContentProvider.TempMediaList Public Property UserExists As Boolean Implements IPluginContentProvider.UserExists Public Property UserSuspended As Boolean Implements IPluginContentProvider.UserSuspended Public Property IsSavedPosts As Boolean Implements IPluginContentProvider.IsSavedPosts Public Property SeparateVideoFolder As Boolean Implements IPluginContentProvider.SeparateVideoFolder Public Property DataPath As String Implements IPluginContentProvider.DataPath Public Property PostsNumberLimit As Integer? Implements IPluginContentProvider.PostsNumberLimit Public Property PostsDateLimit As Date? Implements IPluginContentProvider.PostsDateLimit #End Region #Region "Interface exchange options" Public Sub ExchangeOptionsSet(ByVal Obj As Object) Implements IPluginContentProvider.ExchangeOptionsSet End Sub Public Function ExchangeOptionsGet() As Object Implements IPluginContentProvider.ExchangeOptionsGet Return Nothing End Function #End Region #Region "Interface XML" Public Sub XmlFieldsSet(ByVal Fields As List(Of KeyValuePair(Of String, String))) Implements IPluginContentProvider.XmlFieldsSet End Sub Public Function XmlFieldsGet() As List(Of KeyValuePair(Of String, String)) Implements IPluginContentProvider.XmlFieldsGet Return Nothing End Function #End Region Private Property Responser As Response Public Sub GetMedia() Implements IPluginContentProvider.GetMedia Try If Not Settings.UseM3U8 Then If Settings.FfmpegExists Then LogProvider.Add($"XVIDEOS [{Name}]: The plugin only works with x64 OS.") Else LogProvider.Add($"XVIDEOS [{Name}]: File [ffmpeg.exe] not found") End If Exit Sub End If If Not Responser Is Nothing Then Responser.Dispose() Responser = New Response Responser.Copy(Settings.Responser) Dim NextPage% = 0 Dim r$ Dim j As EContainer, jj As EContainer Dim e As ErrorsDescriber = EDP.ThrowException Dim user$ = Settings.GetUserUrl(Name, False) Dim p As PluginUserMedia Dim EnvirSet As Boolean = False Do Thrower.ThrowAny() r = Responser.GetResponse($"https://www.xvideos.com/{user}/videos/new/{If(NextPage = 0, String.Empty, NextPage)}",, e) If Not r.IsEmptyString Then If Not EnvirSet Then UserExists = True : UserSuspended = False : EnvirSet = True j = JsonDocument.Parse(r).XmlIfNothing With j If .Contains("videos") Then With .Item("videos") If .Count > 0 Then NextPage += 1 For Each jj In .Self p = New PluginUserMedia With { .PostID = jj.Value("id"), .URL = $"https://www.xvideos.com{jj.Value("u")}" } If Not p.PostID.IsEmptyString And Not jj.Value("u").IsEmptyString Then If Not TempPostsList.Contains(p.PostID) Then TempPostsList.Add(p.PostID) : TempMediaList.Add(p) Else Exit Do End If Next End If End With Else .Dispose() Exit Do End If .Dispose() End With Else Exit Do End If Loop If TempMediaList.Count > 0 Then For i% = 0 To TempMediaList.Count - 1 Thrower.ThrowAny() With TempMediaList(i) : TempMediaList(i) = GetVideoData(.URL, Responser, Settings.DownloadUHD.Value, .PostID, LogProvider) : End With Next TempMediaList.RemoveAll(Function(m) m.URL.IsEmptyString) End If Catch oex As OperationCanceledException Catch dex As ObjectDisposedException Catch ex As Exception If Responser.StatusCode = Net.HttpStatusCode.NotFound Then UserExists = False Else LogProvider.Add(ex, "[XVIDEOS.UserData.GetMedia]") End If Finally If TempMediaList.ListExists Then TempMediaList.RemoveAll(Function(m) m.URL.IsEmptyString) End Try End Sub Private Structure VSize : Implements IRegExCreator, IComparable(Of VSize) Friend Size As Integer Friend Value As String Private Function CreateFromArray(ByVal ParamsArray() As String) As Object Implements IRegExCreator.CreateFromArray If ParamsArray.ListExists(2) Then Size = AConvert(Of Integer)(ParamsArray(0), 0) Value = ParamsArray(1) End If Return Me End Function Private Function CompareTo(ByVal Other As VSize) As Integer Implements IComparable(Of VSize).CompareTo Return Size.CompareTo(Other.Size) * -1 End Function End Structure Friend Shared Function GetVideoData(ByVal URL As String, ByVal resp As Response, ByVal DownloadUHD As Boolean, ByVal ID As String, ByRef Logger As ILogProvider) As PluginUserMedia Try If Not URL.IsEmptyString Then Dim r$ = resp.GetResponse(URL,, EDP.ThrowException) If Not r.IsEmptyString Then Dim m$ = RegexReplace(r, M3U8Regex) If Not m.IsEmptyString Then Dim appender$ = RegexReplace(m, M3U8Appender) Dim t$ = RegexReplace(r, VideoTitleRegex) r = resp.GetResponse(m,, EDP.ThrowException) If Not r.IsEmptyString Then Dim ls As List(Of VSize) = RegexFields(Of VSize)(r, {M3U8Reparse}, {1, 2}) If ls.ListExists And Not DownloadUHD Then ls.RemoveAll(Function(v) v.Size > 1080) If ls.ListExists Then ls.Sort() m = $"{appender}/{ls(0).Value}" ls.Clear() Dim pID$ = ID If pID.IsEmptyString Then pID = RegexReplace(r, VideoID) If pID.IsEmptyString Then pID = "0" If Not t.IsEmptyString Then t = t.StringRemoveWinForbiddenSymbols(" ") If t.IsEmptyString Then t = pID Else If t.Length > 100 Then t = Left(t, 100) End If If Not m.IsEmptyString Then Return New PluginUserMedia With { .ContentType = UTypes.m3u8, .PostID = pID, .URL = m, .File = $"{t}.mp4", .SpecialFolder = appender } End If End If End If End If End If End If Return Nothing Catch ex As Exception Logger.Add(ex, $"[XVIDEOS.UserData.GetVideoData({URL})]") Return Nothing End Try End Function Public Sub Download() Implements IPluginContentProvider.Download Try If TempMediaList.Count > 0 Then RaiseEvent TotalCountChanged(TempMediaList.Count - 1) Dim m As PluginUserMedia Dim f As SFile Dim DefPath As String = DataPath.CSFilePSN For i% = 0 To TempMediaList.Count - 1 Thrower.ThrowAny() m = TempMediaList(i) f = m.File f.Path = DefPath m.DownloadState = UStates.Tried Try f = M3U8.Download(m.URL, m.SpecialFolder, Settings.FfmpegFile, f, LogProvider) m.File = f m.DownloadState = UStates.Downloaded Catch ex As Exception m.DownloadState = UStates.Skipped End Try TempMediaList(i) = m RaiseEvent ProgressChanged(1) Next End If Catch oex As OperationCanceledException Catch dex As ObjectDisposedException Catch ex As Exception LogProvider.Add(ex, $"[XVIDEOS.UserData.Download]") End Try End Sub #Region "IDisposable Support" Private disposedValue As Boolean = False Protected Overridable Overloads Sub Dispose(ByVal disposing As Boolean) If Not disposedValue Then If disposing Then If Not Responser Is Nothing Then Responser.Dispose() If TempMediaList.ListExists Then TempMediaList.Clear() If TempPostsList.ListExists Then TempPostsList.Clear() If ExistingContentList.ListExists Then ExistingContentList.Clear() End If disposedValue = True End If End Sub Protected Overrides Sub Finalize() Dispose(False) MyBase.Finalize() End Sub Public Overloads Sub Dispose() Implements IDisposable.Dispose Dispose(True) GC.SuppressFinalize(Me) End Sub #End Region End Class