001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 018package org.apache.commons.net.ftp.parser; 019 020import java.text.ParseException; 021import java.util.List; 022 023import org.apache.commons.net.ftp.FTPClientConfig; 024import org.apache.commons.net.ftp.FTPFile; 025 026/** 027 * Implementation of FTPFileEntryParser and FTPFileListParser for IBM zOS/MVS 028 * Systems. 029 * 030 * @see org.apache.commons.net.ftp.FTPFileEntryParser FTPFileEntryParser (for 031 * usage instructions) 032 */ 033public class MVSFTPEntryParser extends ConfigurableFTPFileEntryParserImpl { 034 035 static final int UNKNOWN_LIST_TYPE = -1; 036 static final int FILE_LIST_TYPE = 0; 037 static final int MEMBER_LIST_TYPE = 1; 038 static final int UNIX_LIST_TYPE = 2; 039 static final int JES_LEVEL_1_LIST_TYPE = 3; 040 static final int JES_LEVEL_2_LIST_TYPE = 4; 041 042 private int isType = UNKNOWN_LIST_TYPE; 043 044 /** 045 * Fallback parser for Unix-style listings 046 */ 047 private UnixFTPEntryParser unixFTPEntryParser; 048 049 /** 050 * Dates are ignored for file lists, but are used for member lists where 051 * possible 052 */ 053 static final String DEFAULT_DATE_FORMAT = "yyyy/MM/dd HH:mm"; // 2001/09/18 054 // 13:52 055 056 /** 057 * Matches these entries: 058 * 059 * <pre> 060 * Volume Unit Referred Ext Used Recfm Lrecl BlkSz Dsorg Dsname 061 * B10142 3390 2006/03/20 2 31 F 80 80 PS MDI.OKL.WORK 062 * </pre> 063 * 064 * @see <a href= 065 * "https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconcepts_159.htm">Data 066 * set record formats</a> 067 */ 068 static final String FILE_LIST_REGEX = "\\S+\\s+" + // volume 069 // ignored 070 "\\S+\\s+" + // unit - ignored 071 "\\S+\\s+" + // access date - ignored 072 "\\S+\\s+" + // extents -ignored 073 // If the values are too large, the fields may be merged (NET-639) 074 "(?:\\S+\\s+)?" + // used - ignored 075 "(?:F|FB|V|VB|U)\\s+" + // recfm - F[B], V[B], U 076 "\\S+\\s+" + // logical record length -ignored 077 "\\S+\\s+" + // block size - ignored 078 "(PS|PO|PO-E)\\s+" + // Dataset organisation. Many exist 079 // but only support: PS, PO, PO-E 080 "(\\S+)\\s*"; // Dataset Name (file name) 081 082 /** 083 * Matches these entries: 084 * <pre> 085 * Name VV.MM Created Changed Size Init Mod Id 086 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001 087 * </pre> 088 */ 089 static final String MEMBER_LIST_REGEX = "(\\S+)\\s+" + // name 090 "\\S+\\s+" + // version, modification (ignored) 091 "\\S+\\s+" + // create date (ignored) 092 "(\\S+)\\s+" + // modification date 093 "(\\S+)\\s+" + // modification time 094 "\\S+\\s+" + // size in lines (ignored) 095 "\\S+\\s+" + // size in lines at creation(ignored) 096 "\\S+\\s+" + // lines modified (ignored) 097 "\\S+\\s*"; // id of user who modified (ignored) 098 099 /** 100 * Matches these entries, note: no header: 101 * <pre> 102 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files 103 * 012345678901234567890123456789012345678901234 104 * 1 2 3 4 105 * </pre> 106 */ 107 static final String JES_LEVEL_1_LIST_REGEX = 108 "(\\S+)\\s+" + // job name ignored 109 "(\\S+)\\s+" + // job number 110 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) 111 "(\\S+)\\s+" + // number of spool files 112 "(\\S+)\\s+" + // Text "Spool" ignored 113 "(\\S+)\\s*" // Text "Files" ignored 114 ; 115 116 /** 117 * JES INTERFACE LEVEL 2 parser 118 * Matches these entries: 119 * <pre> 120 * JOBNAME JOBID OWNER STATUS CLASS 121 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 122 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 123 * </pre> 124 * Sample output from FTP session: 125 * <pre> 126 * ftp> quote site filetype=jes 127 * 200 SITE command was accepted 128 * ftp> ls 129 * 200 Port request OK. 130 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER 131 * JOBNAME JOBID OWNER STATUS CLASS 132 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 133 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 134 * 250 List completed successfully. 135 * ftp> ls job01906 136 * 200 Port request OK. 137 * 125 List started OK for JESJOBNAME=IBMUSER*, JESSTATUS=ALL and JESOWNER=IBMUSER 138 * JOBNAME JOBID OWNER STATUS CLASS 139 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 140 * -------- 141 * ID STEPNAME PROCSTEP C DDNAME BYTE-COUNT 142 * 001 JES2 A JESMSGLG 858 143 * 002 JES2 A JESJCL 128 144 * 003 JES2 A JESYSMSG 443 145 * 3 spool files 146 * 250 List completed successfully. 147 * </pre> 148 */ 149 150 static final String JES_LEVEL_2_LIST_REGEX = 151 "(\\S+)\\s+" + // job name ignored 152 "(\\S+)\\s+" + // job number 153 "(\\S+)\\s+" + // owner ignored 154 "(\\S+)\\s+" + // job status (OUTPUT,INPUT,ACTIVE) ignored 155 "(\\S+)\\s+" + // job class ignored 156 "(\\S+).*" // rest ignored 157 ; 158 159 /* 160 * --------------------------------------------------------------------- 161 * Very brief and incomplete description of the zOS/MVS-file system. (Note: 162 * "zOS" is the operating system on the mainframe, and is the new name for 163 * MVS) 164 * 165 * The file system on the mainframe does not have hierarchal structure as for 166 * example the unix file system. For a more comprehensive description, please 167 * refer to the IBM manuals 168 * 169 * @LINK: 170 * http://publibfp.boulder.ibm.com/cgi-bin/bookmgr/BOOKS/dgt2d440/CONTENTS 171 * 172 * 173 * Dataset names ============= 174 * 175 * A dataset name consist of a number of qualifiers separated by '.', each 176 * qualifier can be at most 8 characters, and the total length of a dataset 177 * can be max 44 characters including the dots. 178 * 179 * 180 * Dataset organisation ==================== 181 * 182 * A dataset represents a piece of storage allocated on one or more disks. 183 * The structure of the storage is described with the field dataset 184 * organinsation (DSORG). There are a number of dataset organisations, but 185 * only two are usable for FTP transfer. 186 * 187 * DSORG: PS: sequential, or flat file PO: partitioned dataset PO-E: 188 * extended partitioned dataset 189 * 190 * The PS file is just a flat file, as you would find it on the unix file 191 * system. 192 * 193 * The PO and PO-E files, can be compared to a single level directory 194 * structure. A PO file consist of a number of dataset members, or files if 195 * you will. It is possible to CD into the file, and to retrieve the 196 * individual members. 197 * 198 * 199 * Dataset record format ===================== 200 * 201 * The physical layout of the dataset is described on the dataset itself. 202 * There are a number of record formats (RECFM), but just a few is relavant 203 * for the FTP transfer. 204 * 205 * Any one beginning with either F or V can safely used by FTP transfer. All 206 * others should only be used with great care. 207 * F means a fixed number of records per 208 * allocated storage, and V means a variable number of records. 209 * 210 * 211 * Other notes =========== 212 * 213 * The file system supports automatically backup and retrieval of datasets. 214 * If a file is backed up, the ftp LIST command will return: ARCIVE Not 215 * Direct Access Device KJ.IOP998.ERROR.PL.UNITTEST 216 * 217 * 218 * Implementation notes ==================== 219 * 220 * Only datasets that have dsorg PS, PO or PO-E and have recfm beginning 221 * with F or V or U, is fully parsed. 222 * 223 * The following fields in FTPFile is used: FTPFile.Rawlisting: Always set. 224 * FTPFile.Type: DIRECTORY_TYPE or FILE_TYPE or UNKNOWN FTPFile.Name: name 225 * FTPFile.Timestamp: change time or null 226 * 227 * 228 * 229 * Additional information ====================== 230 * 231 * The MVS ftp server supports a number of features via the FTP interface. 232 * The features are controlled with the FTP command quote site filetype=<SEQ|JES|DB2> 233 * SEQ is the default and used for normal file transfer JES is used to 234 * interact with the Job Entry Subsystem (JES) similar to a job scheduler 235 * DB2 is used to interact with a DB2 subsystem 236 * 237 * This parser supports SEQ and JES. 238 * 239 * 240 * 241 * 242 * 243 * 244 */ 245 246 /** 247 * The sole constructor for a MVSFTPEntryParser object. 248 * 249 */ 250 public MVSFTPEntryParser() { 251 super(""); // note the regex is set in preParse. 252 super.configure(null); // configure parser with default configurations 253 } 254 255 /** 256 * Parses a line of an z/OS - MVS FTP server file listing and converts it 257 * into a usable format in the form of an <code> FTPFile </code> instance. 258 * If the file listing line doesn't describe a file, then 259 * <code> null </code> is returned. Otherwise a <code> FTPFile </code> 260 * instance representing the file is returned. 261 * 262 * @param entry 263 * A line of text from the file listing 264 * @return An FTPFile instance corresponding to the supplied entry 265 */ 266 @Override 267 public FTPFile parseFTPEntry(final String entry) { 268 if (isType == FILE_LIST_TYPE) { 269 return parseFileList(entry); 270 } else if (isType == MEMBER_LIST_TYPE) { 271 return parseMemberList(entry); 272 } else if (isType == UNIX_LIST_TYPE) { 273 return unixFTPEntryParser.parseFTPEntry(entry); 274 } else if (isType == JES_LEVEL_1_LIST_TYPE) { 275 return parseJeslevel1List(entry); 276 } else if (isType == JES_LEVEL_2_LIST_TYPE) { 277 return parseJeslevel2List(entry); 278 } 279 280 return null; 281 } 282 283 /** 284 * Parse entries representing a dataset list. Only datasets with DSORG PS or 285 * PO or PO-E and with RECFM F[B], V[B], U will be parsed. 286 * 287 * Format of ZOS/MVS file list: 1 2 3 4 5 6 7 8 9 10 Volume Unit Referred 288 * Ext Used Recfm Lrecl BlkSz Dsorg Dsname B10142 3390 2006/03/20 2 31 F 80 289 * 80 PS MDI.OKL.WORK ARCIVE Not Direct Access Device 290 * KJ.IOP998.ERROR.PL.UNITTEST B1N231 3390 2006/03/20 1 15 VB 256 27998 PO 291 * PLU B1N231 3390 2006/03/20 1 15 VB 256 27998 PO-E PLB 292 * 293 * ----------------------------------- Group within Regex [1] Volume [2] 294 * Unit [3] Referred [4] Ext: number of extents [5] Used [6] Recfm: Record 295 * format [7] Lrecl: Logical record length [8] BlkSz: Block size [9] Dsorg: 296 * Dataset organisation. Many exists but only support: PS, PO, PO-E [10] 297 * Dsname: Dataset name 298 * 299 * Note: When volume is ARCIVE, it means the dataset is stored somewhere in 300 * a tape archive. These entries is currently not supported by this parser. 301 * A null value is returned. 302 * 303 * @param entry zosDirectoryEntry 304 * @return null: entry was not parsed. 305 */ 306 private FTPFile parseFileList(final String entry) { 307 if (matches(entry)) { 308 final FTPFile file = new FTPFile(); 309 file.setRawListing(entry); 310 final String name = group(2); 311 final String dsorg = group(1); 312 file.setName(name); 313 314 // DSORG 315 if ("PS".equals(dsorg)) { 316 file.setType(FTPFile.FILE_TYPE); 317 } 318 else if ("PO".equals(dsorg) || "PO-E".equals(dsorg)) { 319 // regex already ruled out anything other than PO or PO-E 320 file.setType(FTPFile.DIRECTORY_TYPE); 321 } 322 else { 323 return null; 324 } 325 326 return file; 327 } 328 329 return null; 330 } 331 332 /** 333 * Parse entries within a partitioned dataset. 334 * 335 * Format of a memberlist within a PDS: 336 * <pre> 337 * 0 1 2 3 4 5 6 7 8 338 * Name VV.MM Created Changed Size Init Mod Id 339 * TBSHELF 01.03 2002/09/12 2002/10/11 09:37 11 11 0 KIL001 340 * TBTOOL 01.12 2002/09/12 2004/11/26 19:54 51 28 0 KIL001 341 * 342 * ------------------------------------------- 343 * [1] Name 344 * [2] VV.MM: Version . modification 345 * [3] Created: yyyy / MM / dd 346 * [4,5] Changed: yyyy / MM / dd HH:mm 347 * [6] Size: number of lines 348 * [7] Init: number of lines when first created 349 * [8] Mod: number of modified lines a last save 350 * [9] Id: User id for last update 351 * </pre> 352 * 353 * @param entry zosDirectoryEntry 354 * @return null: entry was not parsed. 355 */ 356 private FTPFile parseMemberList(final String entry) { 357 final FTPFile file = new FTPFile(); 358 if (matches(entry)) { 359 file.setRawListing(entry); 360 final String name = group(1); 361 final String datestr = group(2) + " " + group(3); 362 file.setName(name); 363 file.setType(FTPFile.FILE_TYPE); 364 try { 365 file.setTimestamp(super.parseTimestamp(datestr)); 366 } catch (final ParseException e) { 367 // just ignore parsing errors. 368 // TODO check this is ok 369 // Drop thru to try simple parser 370 } 371 return file; 372 } 373 374 /* 375 * Assigns the name to the first word of the entry. Only to be used from a 376 * safe context, for example from a memberlist, where the regex for some 377 * reason fails. Then just assign the name field of FTPFile. 378 */ 379 if (entry != null && !entry.trim().isEmpty()) { 380 file.setRawListing(entry); 381 final String name = entry.split(" ")[0]; 382 file.setName(name); 383 file.setType(FTPFile.FILE_TYPE); 384 return file; 385 } 386 return null; 387 } 388 389 /** 390 * Matches these entries, note: no header: 391 * <pre> 392 * [1] [2] [3] [4] [5] 393 * IBMUSER1 JOB01906 OUTPUT 3 Spool Files 394 * 012345678901234567890123456789012345678901234 395 * 1 2 3 4 396 * ------------------------------------------- 397 * Group in regex 398 * [1] Job name 399 * [2] Job number 400 * [3] Job status (INPUT,ACTIVE,OUTPUT) 401 * [4] Number of sysout files 402 * [5] The string "Spool Files" 403 *</pre> 404 * 405 * @param entry zosDirectoryEntry 406 * @return null: entry was not parsed. 407 */ 408 private FTPFile parseJeslevel1List(final String entry) { 409 if (matches(entry)) { 410 final FTPFile file = new FTPFile(); 411 if (group(3).equalsIgnoreCase("OUTPUT")) { 412 file.setRawListing(entry); 413 final String name = group(2); /* Job Number, used by GET */ 414 file.setName(name); 415 file.setType(FTPFile.FILE_TYPE); 416 return file; 417 } 418 } 419 420 return null; 421 } 422 423 /** 424 * Matches these entries: 425 * <pre> 426 * [1] [2] [3] [4] [5] 427 * JOBNAME JOBID OWNER STATUS CLASS 428 * IBMUSER1 JOB01906 IBMUSER OUTPUT A RC=0000 3 spool files 429 * IBMUSER TSU01830 IBMUSER OUTPUT TSU ABEND=522 3 spool files 430 * 012345678901234567890123456789012345678901234 431 * 1 2 3 4 432 * ------------------------------------------- 433 * Group in regex 434 * [1] Job name 435 * [2] Job number 436 * [3] Owner 437 * [4] Job status (INPUT,ACTIVE,OUTPUT) 438 * [5] Job Class 439 * [6] The rest 440 * </pre> 441 * 442 * @param entry zosDirectoryEntry 443 * @return null: entry was not parsed. 444 */ 445 private FTPFile parseJeslevel2List(final String entry) { 446 if (matches(entry)) { 447 final FTPFile file = new FTPFile(); 448 if (group(4).equalsIgnoreCase("OUTPUT")) { 449 file.setRawListing(entry); 450 final String name = group(2); /* Job Number, used by GET */ 451 file.setName(name); 452 file.setType(FTPFile.FILE_TYPE); 453 return file; 454 } 455 } 456 457 return null; 458 } 459 460 /** 461 * preParse is called as part of the interface. Per definition is is called 462 * before the parsing takes place. 463 * Three kind of lists is recognize: 464 * z/OS-MVS File lists 465 * z/OS-MVS Member lists 466 * unix file lists 467 * @since 2.0 468 */ 469 @Override 470 public List<String> preParse(final List<String> orig) { 471 // simply remove the header line. Composite logic will take care of the 472 // two different types of 473 // list in short order. 474 if (orig != null && !orig.isEmpty()) { 475 final String header = orig.get(0); 476 if (header.indexOf("Volume") >= 0 && header.indexOf("Dsname") >= 0) { 477 setType(FILE_LIST_TYPE); 478 super.setRegex(FILE_LIST_REGEX); 479 } else if (header.indexOf("Name") >= 0 && header.indexOf("Id") >= 0) { 480 setType(MEMBER_LIST_TYPE); 481 super.setRegex(MEMBER_LIST_REGEX); 482 } else if (header.indexOf("total") == 0) { 483 setType(UNIX_LIST_TYPE); 484 unixFTPEntryParser = new UnixFTPEntryParser(); 485 } else if (header.indexOf("Spool Files") >= 30) { 486 setType(JES_LEVEL_1_LIST_TYPE); 487 super.setRegex(JES_LEVEL_1_LIST_REGEX); 488 } else if (header.indexOf("JOBNAME") == 0 489 && header.indexOf("JOBID") > 8) {// header contains JOBNAME JOBID OWNER // STATUS CLASS 490 setType(JES_LEVEL_2_LIST_TYPE); 491 super.setRegex(JES_LEVEL_2_LIST_REGEX); 492 } else { 493 setType(UNKNOWN_LIST_TYPE); 494 } 495 496 if (isType != JES_LEVEL_1_LIST_TYPE) { // remove header is necessary 497 orig.remove(0); 498 } 499 } 500 501 return orig; 502 } 503 504 /** 505 * Explicitly set the type of listing being processed. 506 * @param type The listing type. 507 */ 508 void setType(final int type) { 509 isType = type; 510 } 511 512 /* 513 * @return 514 */ 515 @Override 516 protected FTPClientConfig getDefaultConfiguration() { 517 return new FTPClientConfig(FTPClientConfig.SYST_MVS, 518 DEFAULT_DATE_FORMAT, null); 519 } 520 521}