Subversion Repository Public Repository

CommonUtility

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
´╗┐using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using WinSCP;
using System.Data.Linq;
using System.Configuration;
using STP.Database.Report;
using System.Text.RegularExpressions;
using System.Threading;
using System.IO;

namespace COMMONUTIL.FTP
{
    public class FTPUtility
    {
        private static readonly log4net.ILog log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
        private string _connectionString;
        private Session _session;

        public FTPUtility(string connectionString, string FTPHost, string username)
        {
            _connectionString = connectionString;

            if (_session == null)
            {
                _session = new Session();
                _session.Open(GetSessionOptions(FTPHost, username));
            }
        }

        public void GetFile(string remoteFilePath, string localFilePath, bool delete_after)
        {
            try
            {
                Console.WriteLine("\n FTPUTIL: Get File: " + remoteFilePath);

                // Upload files
                TransferOptions transferOptions = new TransferOptions();
                transferOptions.TransferMode = TransferMode.Binary;
                transferOptions.PreserveTimestamp = true;

                _session.GetFiles(remoteFilePath, localFilePath, delete_after, transferOptions).Check();

            }
            catch (Exception e)
            {
                log.Error("Exception retrieving file: " + remoteFilePath + ". \n Exception: " + e.Message);
                _session.Dispose();
                throw new Exception("FTPUtility EXCEPTION(GetFile()): " + e.Message);
            }
        }

        public List<string> ListFilesInDirectory(string remoteFilePath)
        {
            List<string> fileList = new List<string>();

            try
            {
                Console.WriteLine("\n FTPUTIL: Get File: " + remoteFilePath);

                // Upload files
                TransferOptions transferOptions = new TransferOptions();
                transferOptions.TransferMode = TransferMode.Binary;
                transferOptions.PreserveTimestamp = true;

                RemoteDirectoryInfo directory = _session.ListDirectory(remoteFilePath);

                foreach (RemoteFileInfo file in directory.Files)
                    fileList.Add(file.Name);
                
            }
            catch (Exception e)
            {
                log.Error("Exception listing files in directory: " + remoteFilePath + ". \n Exception: " + e.Message);
                _session.Dispose();
                throw new Exception("FTPUtility EXCEPTION(ListFilesInDirectory()): " + e.Message);
            }

            return fileList;
        }

        public void PutFile(string remoteFilePath, string localFilepath, bool delete_after)
        {
            try
            {

                Console.WriteLine("\n FTPUTIL: Put File: " + remoteFilePath);

                // Upload files
                TransferOptions transferOptions = new TransferOptions();
                transferOptions.TransferMode = TransferMode.Binary;
                transferOptions.ResumeSupport.State = TransferResumeSupportState.Off;

                _session.PutFiles(localFilepath, remoteFilePath, false, transferOptions).Check();
                
            }
            catch (Exception e)
            {
                log.Error("Exception putting file: " + remoteFilePath + ".\n  Exception: " + e.Message);
                _session.Dispose();
                throw new Exception("FTPUtility EXCEPTION(PutFile()): " + e.Message);
            } 
        }

        public void MoveFile(string sourceFilePath, string targetFilePath, bool delete_after)
        {
           
            try
            {
                Console.WriteLine("\n FTPUTIL: Move File: " + targetFilePath);

                // Upload files
                TransferOptions transferOptions = new TransferOptions();
                transferOptions.TransferMode = TransferMode.Binary;

                _session.MoveFile(sourceFilePath, targetFilePath);

                if (delete_after)
                    _session.RemoveFiles(sourceFilePath);

            }
            catch (Exception e)
            {
                log.Error("Exception putting file: " + targetFilePath + ".\n Exception: " + e.Message);
                _session.Dispose();
                throw new Exception("FTPUtility EXCEPTION(MoveFile()): " + e.Message);
            }
            
        }

        public bool DirExists(string remoteDirectory)
        {
            bool ret = false;

            try
            {
                // Upload files
                TransferOptions transferOptions = new TransferOptions();
                transferOptions.TransferMode = TransferMode.Binary;

                ret = _session.FileExists(remoteDirectory);
               
            }
            catch (Exception e)
            {
                log.Error("Exception checking if directory exists: " + remoteDirectory + ".\n  Exception: " + e.Message);
                _session.Dispose();
                throw new Exception("FTPUtility EXCEPTION(DirExists()): " + e.Message);
            }

            return ret;
        }

        public void CreateDir(string remoteDirectory)
        {
            try
            {
                // Upload files
                TransferOptions transferOptions = new TransferOptions();
                transferOptions.TransferMode = TransferMode.Binary;

                _session.CreateDirectory(remoteDirectory);
                
            }
            catch (Exception e)
            {
                log.Error("Exception creating directory: " + remoteDirectory + ".\n  Exception: " + e.Message);
                _session.Dispose();
                throw new Exception("FTPUtility EXCEPTION(CreateDir()): " + e.Message);
            }

        }

        public bool FileExists(string remoteFilePath)
        {

            Console.WriteLine("\n FTPUTIL: File exists check: " + remoteFilePath);
            bool ret = false;

            try
            {
                // Upload files
                TransferOptions transferOptions = new TransferOptions();
                transferOptions.TransferMode = TransferMode.Binary;

                ret = _session.FileExists(remoteFilePath);

                Console.WriteLine("\n FileExists SUCCESS");
            }
            catch (Exception e)
            {
                log.Error("Exception checking if file exists at location: " + remoteFilePath + ".\n  Exception: " + e.Message);
                _session.Dispose();
                throw new Exception("FTPUtility EXCEPTION(FileExists()): " + e.Message);
            }

            return ret;
        }

        /// <summary>
        /// Check if file exists (partial file name - i.e. files with timestamps) 
        /// Get all directory files and see if partial filename matches any files
        /// </summary>
        /// <param name="remoteDirectory">Remote FTP directory</param>
        /// <param name="filename_pattern">Remote partial filename i.e. bw_pncdaily_20150129_.*\.txt</param>
        /// <returns></returns>
        public string FileExists(string remoteDirectory, string filename_pattern)
        {
            string filename = "";

            // Upload files
            TransferOptions transferOptions = new TransferOptions();
            transferOptions.TransferMode = TransferMode.Binary;

            RemoteDirectoryInfo directory_info = _session.ListDirectory(remoteDirectory);
            Regex reg = new Regex(filename_pattern);

            foreach (RemoteFileInfo fileinfo in directory_info.Files)
            {
                if (reg.IsMatch(fileinfo.Name))
                    return fileinfo.Name;
            }

            return filename;
        }

        /// <summaryi
        /// Check if a remote file exists
        /// </summary>
        /// <param name="remoteFilePath">Full file path</param>
        /// <returns>bool</returns>
        public bool FileExists(string remoteFilePath, string client_id, string filename, string remoteDirectory)
        {
            bool ret = false;

            // Upload files
            TransferOptions transferOptions = new TransferOptions();
            transferOptions.TransferMode = TransferMode.Binary;

            ret = _session.FileExists(remoteFilePath);

            if (!ret && client_id.Equals("BW") && filename.StartsWith("bw_pncdaily"))
            {
                string most_recent_filename = GetMostRecentFile(remoteDirectory);
                ret = _session.FileExists(remoteDirectory + "/" + most_recent_filename);
            }
            

            return ret;
        }

        public string GetMostRecentFile(string remoteDirectory)
        {
            RemoteFileInfo latest_file = null;

            // Upload files
            TransferOptions transferOptions = new TransferOptions();
            transferOptions.TransferMode = TransferMode.Binary;

            RemoteDirectoryInfo directory_info = _session.ListDirectory(remoteDirectory);

            int count = 0;
            DateTime lastwritetime = new DateTime();

            foreach (RemoteFileInfo fileinfo in directory_info.Files)
            {
                count++;

                if (count == 1)
                {
                    lastwritetime = fileinfo.LastWriteTime;
                    latest_file = fileinfo;
                }

                DateTime this_file_lastwritetime = fileinfo.LastWriteTime;

                if (this_file_lastwritetime > lastwritetime)
                {
                    lastwritetime = this_file_lastwritetime;
                    latest_file = fileinfo;
                }
            }
            

            return latest_file.Name;
        }

        /// <summary>
        /// Poll for a file on an FTP server until its fully transferred on source 
        /// large files may take some time upload on server - this eliminates partial file pulls
        /// </summary>
        /// <param name="remoteFilePath">Remote FTP File Path</param>
        /// <param name="localFilePath">Local File Path to store file</param>
        /// <param name="FTPHost">FTP Host / IP address</param>
        /// <param name="username">FTP username</param>
        /// <param name="delete_after">delete file  on source after get</param>
        public void GetFullFile(string remoteFilePath, string localFilePath, bool delete_after)
        {
            try
            {
                Console.WriteLine("\n FTPUTIL: Get Full File: " + remoteFilePath);

                int previous_filesize = 0;
                int current_filesize = 1;
                int max_retry = 10;
                int current_try = 1;

                if (FileExists(remoteFilePath)) // only check if file already exists
                {
                    // Upload files
                    TransferOptions transferOptions = new TransferOptions();
                    transferOptions.TransferMode = TransferMode.Binary;
                    transferOptions.PreserveTimestamp = true;

                    while (current_try <= max_retry)  // Max 10 Trys
                    {
                        _session.GetFiles(remoteFilePath, localFilePath, delete_after, transferOptions).Check();
                        byte[] current_file = File.ReadAllBytes(localFilePath);
                        current_filesize = current_file.Length;

                        if (previous_filesize == current_filesize && current_filesize != 0)
                            break;

                        current_try++;
                        previous_filesize = current_filesize;
                        Thread.Sleep(1000); // Wait 3 seconds to try again.
                    }
                }
                
            }
            catch (Exception e)
            {
                log.Error("Exception retrieving full file: " + remoteFilePath + ". \n Exception: " + e.Message);
                _session.Dispose();
                throw new Exception("FTPUtility EXCEPTION(GetFile()): " + e.Message);
            }
        }

        //gets last modified file with naming convention provided
        public string GetFilename(string remoteFilePath, string file_name_pattern)
        {

            Console.WriteLine("\n FTPUTIL: Get Filename: " + remoteFilePath);
            string filename = "";

            try
            {
                // Upload files
                TransferOptions transferOptions = new TransferOptions();
                transferOptions.TransferMode = TransferMode.Binary;

                RemoteDirectoryInfo directory_info = _session.ListDirectory(remoteFilePath);

                DateTime last_modified = new DateTime();
                foreach (RemoteFileInfo fileinfo in directory_info.Files)
                {
                    if (fileinfo.Name.Contains(file_name_pattern) && fileinfo.LastWriteTime > last_modified)
                    {
                        filename = fileinfo.Name;
                        last_modified = fileinfo.LastWriteTime;
                    }
                }
                
                Console.WriteLine("\n FileExists SUCCESS");
            }
            catch (Exception e)
            {
                log.Error("Exception checking if file exists at location: " + remoteFilePath + ".\n  Exception: " + e.Message);
                _session.Dispose();
                throw new Exception("FTPUtility EXCEPTION(FileExists()): " + e.Message);
            }

            return filename;
        }


        //gets last modified file with naming convention provided
        public DateTime GetFileLastModifiedDateTime(string remoteFilePath, string file_name_pattern)
        {

            Console.WriteLine("\n FTPUTIL: GetFileLastModifiedDateTime: " + remoteFilePath);
            string filename = "";
            DateTime last_modified = new DateTime();

            try
            {
                // Upload files
                TransferOptions transferOptions = new TransferOptions();
                transferOptions.TransferMode = TransferMode.Binary;

                RemoteDirectoryInfo directory_info = _session.ListDirectory(remoteFilePath);

                foreach (RemoteFileInfo fileinfo in directory_info.Files)
                {
                    if (fileinfo.Name.Contains(file_name_pattern) && fileinfo.LastWriteTime > last_modified)
                    {
                        filename = fileinfo.Name;
                        last_modified = fileinfo.LastWriteTime;
                    }
                }
                
                Console.WriteLine("\n FileExists SUCCESS");
            }
            catch (Exception e)
            {
                log.Error("Exception checking if file exists at location: " + remoteFilePath + ". \n Exception: " + e.Message);
                _session.Dispose();
                throw new Exception("FTPUtility EXCEPTION(FileExists()): " + e.Message);
            }

            return last_modified;
        }

        public SessionOptions GetSessionOptions(string FTPHost, string username)
        {
            SessionOptions sessionOptions = new SessionOptions();

            event_logDataContext db = new event_logDataContext(_connectionString);

            try
            {
                ISingleResult<proc_get_ftp_server_info_by_host_and_userResult> result = db.proc_get_ftp_server_info_by_host_and_user(FTPHost, "", username);
                foreach (proc_get_ftp_server_info_by_host_and_userResult r in result)
                {

                    string host = (r.ip != null && !r.ip.Trim().Equals("")) ? r.ip : r.url;

                    sessionOptions.Protocol = (r.secure.Equals('Y')) ? Protocol.Sftp : Protocol.Ftp;
                    sessionOptions.HostName = host;
                    sessionOptions.UserName = r.username;
                    sessionOptions.Password = r.pwd;

                    if (r.port != null && r.port > 0)
                    {
                        sessionOptions.PortNumber = r.port.Value;
                        sessionOptions.SshHostKeyFingerprint = r.SshHostKeyFingerprint;
                    }
                    else
                    {
                        if (sessionOptions.Protocol == Protocol.Ftp)
                            sessionOptions.PortNumber = 21;
                        else if (sessionOptions.Protocol == Protocol.Sftp)
                        {
                            sessionOptions.PortNumber = 22;
                            sessionOptions.SshHostKeyFingerprint = r.SshHostKeyFingerprint;
                        }
                    }
                    break;
                }
            }
            catch (Exception e)
            {
                log.Error("Exception getting session settings. \n FTP Host: " + FTPHost + "\n Username: " + username + " \n Exception: " + e.Message);
                throw new Exception("FTPUtility EXCEPTION(GetSessionOptions()): " + e.Message);
            }

            return sessionOptions;
        }

        public void CloseSession()
        {
            if (_session != null && _session.Opened)
             _session.Dispose();
        }
    }
}

Commits for CommonUtility/CommonUtility/CommonUtility/FTPUtility.cs

Diff revisions: vs.
Revision Author Commited Message
19 Diff Diff jdomsky picture jdomsky Fri 06 Jan, 2017 19:03:15 +0000
18 Diff Diff jdomsky picture jdomsky Wed 16 Nov, 2016 22:21:31 +0000

fingerprint needed on different ports

15 Diff Diff csallemi picture csallemi Thu 21 Jul, 2016 14:35:45 +0000

updated methods to ftp helper

14 Diff Diff csallemi picture csallemi Mon 18 Jul, 2016 18:30:39 +0000

close session method update

13 Diff Diff csallemi picture csallemi Mon 18 Jul, 2016 18:25:54 +0000

session added to constructor

12 Diff Diff csallemi picture csallemi Mon 18 Jul, 2016 18:22:43 +0000

updated constructor to include session

11 Diff Diff jdomsky picture jdomsky Mon 18 Jul, 2016 16:09:31 +0000
8 Diff Diff csallemi picture csallemi Mon 20 Jun, 2016 19:19:40 +0000

Get Full File FTPUtil – make sure filesize is not 0

7 Diff Diff csallemi picture csallemi Mon 06 Jun, 2016 20:56:49 +0000

GetFullFile Polling Mechanism

6 csallemi picture csallemi Thu 02 Jun, 2016 03:42:27 +0000

file exists with pattern