TacklerTxns.scala 10.5 KB
Newer Older
35V LG84's avatar
35V LG84 committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
/*
 * Copyright 2016-2018 E257.FI
 *
 * git2Txns is based on example
 * by: Copyright 2013, 2014 Dominik Stadler
 * license: Apache License v2.0
 * url: https://raw.githubusercontent.com/centic9/jgit-cookbook/
 * commit: 276ad0fecb4f1c616ef459ed8b7feb6d503724eb
 * file: jgit-cookbook/src/main/java/org/dstadler/jgit/api/ReadFileFromCommit.java
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package fi.e257.tackler.parser
import java.nio.file.Path

35V LG84's avatar
35V LG84 committed
27
import better.files._
35V LG84's avatar
35V LG84 committed
28
import cats.implicits._
29
import fi.e257.tackler.api.GitInputReference
35V LG84's avatar
35V LG84 committed
30 31
import fi.e257.tackler.core.{Settings, TacklerException}
import fi.e257.tackler.model.{OrderByTxn, TxnData, Txns}
32 33
import org.eclipse.jgit.lib.{FileMode, ObjectId, Repository}
import org.eclipse.jgit.revwalk.{RevCommit, RevWalk}
35V LG84's avatar
35V LG84 committed
34 35 36 37
import org.eclipse.jgit.storage.file.FileRepositoryBuilder
import org.eclipse.jgit.treewalk.TreeWalk
import org.eclipse.jgit.treewalk.filter.{AndTreeFilter, PathFilter, PathSuffixFilter}
import org.slf4j.{Logger, LoggerFactory}
35V LG84's avatar
35V LG84 committed
38 39

import scala.util.control.NonFatal
35V LG84's avatar
35V LG84 committed
40
import fi.e257.tackler.Scala12to13.Converters._
35V LG84's avatar
35V LG84 committed
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56

/**
 * Helper methods for [[TacklerTxns]] and Txns Input handling.
 */
object TacklerTxns {
  type GitInputSelector = Either[String, String]

  private val log: Logger = LoggerFactory.getLogger(this.getClass)

  /**
   * Get input Txns paths based on configuration settings
   *
   * @param settings of base dir path and glob configuration
   * @return sequence of input txn pahts
   */
  def inputPaths(settings: Settings): Seq[Path] = {
57 58
    log.info("FS: dir = {}", settings.input_fs_dir.toString)
    log.info("FS: glob = {}", settings.input_fs_glob)
35V LG84's avatar
35V LG84 committed
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122

    File(settings.input_fs_dir)
      .glob(settings.input_fs_glob)(visitOptions = File.VisitOptions.follow)
      .map(f => f.path)
      .toSeq
  }

  /**
   * Make git commit id based input selector.
   *
   * @param commitId as string
   * @return git input selector for commitId
   */
  def gitCommitId(commitId: String): GitInputSelector = {
    Right[String, String](commitId)
  }

  /**
   * Make git reference based input selector from settings
   *
   * @param settings containing git reference config
   * @return git input selector for reference
   */
  @SuppressWarnings(Array("org.wartremover.warts.Overloading"))
  def gitReference(settings: Settings) :  GitInputSelector = {
    Left[String, String](settings.input_git_ref)
  }

  /**
   * Make git reference based input selector
   *
   * @param reference git reference as string
   * @return git input selector for reference
   */
  @SuppressWarnings(Array("org.wartremover.warts.Overloading"))
  def gitReference(reference: String) :  GitInputSelector = {
    Left[String, String](reference)
  }

}

/**
 * Generate Transactions from selected inputs.
 *
 * These take an input(s) as argument and
 * returns sequence of transactions.
 *
 * If there is an error, they throw an exception.
 *
 * @param settings to control how inputs and txns are handled
 */
class TacklerTxns(val settings: Settings) extends CtxHandler {
  private val log: Logger = LoggerFactory.getLogger(this.getClass)

  /**
   * Get Transactions from list of input paths.
   * Throws an exception in case of error.
   * See also [[TacklerTxns.inputPaths]]
   *
   * @param paths input as seq of files
   * @return TxnData
   */
  def paths2Txns(paths: Seq[Path]): TxnData = {

123
    TxnData(None,
35V LG84's avatar
35V LG84 committed
124
      paths.par.flatMap(inputPath => {
35V LG84's avatar
35V LG84 committed
125
        try {
126
          log.trace("FS: handle file = {}", inputPath.toString)
35V LG84's avatar
35V LG84 committed
127 128 129 130
          val txnsCtx = TacklerParser.txnsFile(inputPath)
          handleTxns(txnsCtx)
        } catch {
          case ex: Exception => {
131 132 133 134
            log.error(
              "FS: Error while processing file" + "\n" +
              "   path = " + inputPath.toString + "\n" +
              "   msg: " + ex.getMessage)
35V LG84's avatar
35V LG84 committed
135 136 137
            throw ex
          }
        }
35V LG84's avatar
35V LG84 committed
138 139 140
      }).seq.sorted(OrderByTxn),
      Some(settings)
    )
35V LG84's avatar
35V LG84 committed
141 142
  }

35V LG84's avatar
35V LG84 committed
143 144 145 146 147 148 149
  /**
   * Get commit id based on input ref
   *
   * @return git commit id (as ObjectId)
   */
  private def getCommitId(repository: Repository, inputSelector: TacklerTxns.GitInputSelector): ObjectId = {
    inputSelector match {
150
      case Left(refStr) => {
35V LG84's avatar
35V LG84 committed
151 152 153 154
        log.info("GIT: reference = {}", refStr)

        val refOpt = Option(repository.findRef(refStr))
        val ref = refOpt.getOrElse({
155 156 157
          val msg = "GIT: ref not found or it is invalid: [" + refStr + "]"
          log.error(msg)
          throw new TacklerException(msg)
35V LG84's avatar
35V LG84 committed
158 159
        })
        ref.getObjectId
160 161
      }
      case Right(commitIdStr) => {
35V LG84's avatar
35V LG84 committed
162 163 164 165 166 167
        log.info("GIT: commitId = {}", commitIdStr)
        try {
          // resolve fails either with null or exceptions
          Option(repository.resolve(commitIdStr))
            .getOrElse({
              // test: uuid: 7cb6af2e-3061-4867-96e3-ee175b87a114
168
              val msg = "GIT: Can not resolve given id: [" + commitIdStr + "]"
35V LG84's avatar
35V LG84 committed
169 170 171 172 173
              log.error(msg)
              throw new TacklerException(msg)
            })
        } catch {
          case e: RuntimeException =>
174
            val msg = "GIT: Can not resolve commit by given id: [" + commitIdStr + "], Message: [" + e.getMessage + "]"
35V LG84's avatar
35V LG84 committed
175 176 177 178
            log.error(msg)
            throw new TacklerException(msg)
        }
      }
179 180 181 182 183 184
    }
  }

  /**
   * Get Git repository as managed resource.
   * Repository must be bare.
35V LG84's avatar
35V LG84 committed
185
   * Caller must close repository after use
186 187
   *
   * @param gitdir path/to/repo.git
35V LG84's avatar
35V LG84 committed
188
   * @return repository
189
   */
35V LG84's avatar
35V LG84 committed
190
  private def getRepo(gitdir: File): Repository = {
191 192
    log.info("GIT: repo = {}", gitdir.toString())
    try {
35V LG84's avatar
35V LG84 committed
193
      (new FileRepositoryBuilder)
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
        .setGitDir(gitdir.toJava)
        .setMustExist(true)
        .setBare()
        .build()
    } catch {
      case e: org.eclipse.jgit.errors.RepositoryNotFoundException => {
        val msg = "GIT: Repository not found\n" +
          "   Could not find usable git repository, check repository path.\n" +
          "   Make sure repository is bare or path is pointing to .git directory.\n" +
          "   Message: " + e.getMessage
        log.error(msg)
        throw new TacklerException(msg)
      }
    }
  }

  /**
   * Get Transactions from GIT based storage.
   * Basic git repository information is read from settings,
   * but input setting (ref or commit id) is an argument.
   * Throws an exception in case of error.
   * See [[TacklerTxns.gitCommitId]] et.al.
   *
   * feature: 06b4a9b1-f48c-4b33-8811-1f32cdc44d7b
   * coverage: "sorted" tested by 1d2c22c1-e3fa-4cd4-a526-45318c15d13e
   *
   * @param inputRef Left(ref) or Right(commitId)
   * @return TxnData
   */
35V LG84's avatar
35V LG84 committed
223 224 225
  @SuppressWarnings(Array(
    "org.wartremover.warts.TraversableOps",
    "org.wartremover.warts.Equals"))
226 227
  def git2Txns(inputRef: TacklerTxns.GitInputSelector): TxnData = {

35V LG84's avatar
35V LG84 committed
228
    using(getRepo(settings.input_git_repository))(repository => {
229 230 231 232 233

      val gitdir = settings.input_git_dir
      val suffix = settings.input_git_suffix

      val commitId = getCommitId(repository, inputRef)
35V LG84's avatar
35V LG84 committed
234

35V LG84's avatar
35V LG84 committed
235 236
      using(new RevWalk(repository))(revWalk => {
        // a RevWalk allows to walk over commits based on defined filtering
35V LG84's avatar
35V LG84 committed
237

238
        val commit: RevCommit = try {
35V LG84's avatar
35V LG84 committed
239 240 241
          revWalk.parseCommit(commitId)
        } catch {
          case e: org.eclipse.jgit.errors.MissingObjectException =>
242
            val msg = "GIT: Can not find commit by given id: [" + commitId.getName + "], Message: [" + e.getMessage + "]"
35V LG84's avatar
35V LG84 committed
243 244 245 246
            log.error(msg)
            throw new TacklerException(msg)
        }

247 248 249 250
        log.info("GIT: commit = " + commit.getName)
        log.info("GIT: dir = " + gitdir)
        log.info("GIT: suffix = " + suffix)

35V LG84's avatar
35V LG84 committed
251 252 253
        val tree = commit.getTree

        // now try to find files
35V LG84's avatar
35V LG84 committed
254
        using(new TreeWalk(repository))(treeWalk => {
35V LG84's avatar
35V LG84 committed
255 256 257 258
          treeWalk.addTree(tree)
          treeWalk.setRecursive(true)

          treeWalk.setFilter(AndTreeFilter.create(
259 260
            PathFilter.create(gitdir),
            PathSuffixFilter.create(suffix)))
35V LG84's avatar
35V LG84 committed
261 262

          // Handle files
35V LG84's avatar
35V LG84 committed
263
          val rawTxns = (for {
35V LG84's avatar
35V LG84 committed
264 265 266 267
            _ <- Iterator.continually(treeWalk.next()).takeWhile(p => p === true)
          } yield {
            val objectId = treeWalk.getObjectId(0)
            if (FileMode.REGULAR_FILE.equals(treeWalk.getFileMode(0))) {
268 269 270 271
              try {
                gitObject2Txns(repository, objectId)
              } catch {
                case NonFatal(ex) => {
272
                  val msg = "GIT: Error while processing git object" + "\n" +
273 274 275 276 277 278 279 280
                    "   commit id: " + commit.getName + "\n" +
                    "   object id: " + objectId.getName + "\n" +
                    "   path: " + treeWalk.getPathString + "\n" +
                    "   msg : " + ex.getMessage
                  log.error(msg)
                  throw new TacklerException(msg)
                }
              }
35V LG84's avatar
35V LG84 committed
281
            } else {
282
              val msg = "GIT: Found matching object, but it is not regular file" + "\n" +
35V LG84's avatar
35V LG84 committed
283 284
                "   commit id: " + commit.getName + "\n" +
                "   object id: " + objectId.getName + "\n" +
285
                "   path: " + treeWalk.getPathString
35V LG84's avatar
35V LG84 committed
286 287 288
              log.error(msg)
              throw new TacklerException(msg)
            }
289
          }).toSeq
35V LG84's avatar
35V LG84 committed
290

291
          val meta = GitInputReference(
35V LG84's avatar
35V LG84 committed
292 293
            commit.getName,
            inputRef.left.toOption,
294 295
            gitdir,
            suffix,
35V LG84's avatar
35V LG84 committed
296 297 298
            commit.getShortMessage
          )

299
          TxnData(Some(meta), rawTxns.flatten.sorted(OrderByTxn), Some(settings))
35V LG84's avatar
35V LG84 committed
300 301 302
        })
      })
    })
35V LG84's avatar
35V LG84 committed
303 304
  }

305 306
  private def gitObject2Txns(repository: Repository, objectId: ObjectId): Txns = {

307
    log.trace("GIT: handle object id = {}", objectId.getName)
308

35V LG84's avatar
35V LG84 committed
309
    val loader = repository.open(objectId, org.eclipse.jgit.lib.Constants.OBJ_BLOB)
310

35V LG84's avatar
35V LG84 committed
311
    using(loader.openStream())(stream => {
312
      handleTxns(TacklerParser.txnsStream(stream))
35V LG84's avatar
35V LG84 committed
313
    })
314
  }
315

35V LG84's avatar
35V LG84 committed
316 317 318 319 320 321 322 323 324 325 326 327 328
  /**
   * Parse and converts input string to Txns
   * Throws an exception in case of error.
   *
   * feature: a94d4a60-40dc-4ec0-97a3-eeb69399f01b
   * coverage: "sorted" tested by 200aad57-9275-4d16-bdad-2f1c484bcf17
   *
   * @param input as text
   * @return TxnData
   */
  def string2Txns(input: String): TxnData = {

    val txnsCtx = TacklerParser.txnsText(input)
329
    TxnData(None, handleTxns(txnsCtx).sorted(OrderByTxn), Some(settings))
35V LG84's avatar
35V LG84 committed
330 331
  }
}