Commit 7a17324a authored by Cedric lardeux 's avatar Cedric lardeux

update script and tutorial

parent 71fbf871
......@@ -4,6 +4,16 @@ This repository contain QGIS processing scripts
Alpha version, works on going !
# How to install ?
- Download all the files and subfolder in an empty directory
- Open a terminal
- go to the directory using command like "cd /home/myuser/MyFolder
- type the command "sh FullInstall.sh
- It will ask you to give the sudo password
- Wait
- At the end reboot the computer
- Enjoy !!!
## Authors
- Cedric Lardeux
......
##Sentinel-1 Deforestation Process=group
##Sentinel-1 IW Additional Batch Processing=group
##Clip Rasters from folder to other one=name
##Input_Data_Folder=folder
##Input_Polygon_File=vector
......
ENVI
samples = 1441
lines = 721
bands = 1
header offset = 24
file type = ENVI Standard
data type = 4
interleave = bsq
sensor type = Unknown
byte order = 1
wavelength units = Unknown
# Introduction
# Different server
## ASF Vertex
- Download the metaling
- Run this command
aria2c --http-auth-challenge=true --http-user=yourlogin --http-passwd=yourpassword ASF_20180406-131333.metalink
This diff is collapsed.
Ce git contient des codes de traitement de données Sentinel-1 spécifiquement mis à jour pour la conférence FOSS4G-FR 2018
This diff is collapsed.
##Sentinel-1 IW GRD Batch Processing=group
##2 - Adaptative Temporal Filter=name
##Input_Data_Folder=folder
##ParameterNumber|Spatial_Window_Size_for_Temporal_Filter|Window size for temporal filtering|3|21|11|False
##ParameterNumber|Temporal_Change_Thresold|Change threshold for temporal filtering. 0 for Quegan equivalent|0.|1.|0.97|False
##*ParameterBoolean|Apply_Lee_Pre_Filtering|Apply Lee pre filtering|False|False
##*ParameterNumber|Spatial_Window_Size_for_Lee_Filter|Window size for Lee filtering|3|21|5|False
##*ParameterNumber|Looks_Number_for_Lee_Filter|Number of look to use for lee filtering|1|100|5|False
##OutputDirectory|Output_Data_Folder|Set the directory where the the files will be saved
##*ParameterBoolean|Output_in_dB|Output in dB|True|False
##Ram=number 512
'''
Output_in_dB=boolean True
'''
import os, sys, glob, re, subprocess
from qgis.core import QgsApplication
QgisSettingPath = QgsApplication.qgisSettingsDirPath()
ScriptPath = os.path.join(QgisSettingPath, 'processing/scripts/S1AdaptTemporalFiltering.py')
ScriptPath = ScriptPath.replace("\\","/")
cmd = "python "
cmd += ScriptPath
cmd += ' -InDir ' + Input_Data_Folder
cmd += ' -WinTempFilt ' + str(Spatial_Window_Size_for_Temporal_Filter)
cmd += ' -AdaptTempChangeThresch ' + str(Temporal_Change_Thresold)
cmd += ' -ApplyLeePreFilt ' + str(Apply_Lee_Pre_Filtering)
cmd += ' -WinLeeFilt ' + str(Spatial_Window_Size_for_Lee_Filter)
cmd += ' -ENLLeeFilt ' + str(Looks_Number_for_Lee_Filter)
cmd += ' -OutputIndB ' + str(Output_in_dB)
cmd += ' -OutDir ' + Output_Data_Folder
cmd += ' -Ram ' + str(Ram)
p1 = subprocess.Popen (cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret= p1.communicate()[1]
This diff is collapsed.
##Sentinel-1 IW Batch Processing=group
##GRD Products Dual Polar Color Composition=name
##Sentinel-1 IW GRD Batch Processing=group
##Generate Dual Pol Color Composition=name
##Input_Data_Folder=folder
##Output_Data_Folder=folder
##Ram=number 256
......
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 26 18:13:52 2017
S1OTBLibrary.py
@author: cedric
"""
'''
Sentinel-1 OTB Library
'''
This diff is collapsed.
# -*- coding: utf-8 -*-
"""
Created on Thu Oct 5 12:46:00 2017
@author: cedric
"""
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 26 18:14:51 2017
@author: cedric
"""
'''
This script apply calibration and orthorectification process of S1 GRD data
'''
'''
IMPORT
'''
import os, sys
from inspect import getsourcefile
import logging
from datetime import datetime
'''
This function join path always using unix sep
'''
def modOsJoinPath(alistToJoin):
joinedPath = os.path.join(*alistToJoin).replace("\\","/")
return joinedPath
'''
This function return the qgis script folder
'''
def getQGISProcessingScriptFolder():
from qgis.core import QgsApplication
QGISProcessingScriptFolder = os.path.dirname(QgsApplication.qgisSettingsDirPath())
QGISProcessingScriptFolder = modOsJoinPath([QGISProcessingScriptFolder,
'processing', 'scripts'])
return QGISProcessingScriptFolder
'''
This function load the necessary libs in different way
if we are running script in qgis processing or not
'''
def AddS1LibToPath():
# Boolean to know if in qgis
import qgis.utils
inqgis = qgis.utils.iface is not None
if inqgis:
QGISProcessingScriptFolder = getQGISProcessingScriptFolder()
# Create the S1Lib lib folder path
ScriptPath = modOsJoinPath([QGISProcessingScriptFolder, 'S1Lib'])
else:
LocalScriptFileDir = os.path.dirname(os.path.abspath((getsourcefile(lambda:0)))).replace("\\","/")
# Create the S1Lib lib folder path
ScriptPath = modOsJoinPath([LocalScriptFileDir, 'S1Lib'])
# Add path to sys
sys.path.insert(0,ScriptPath)
# Load OTB Libs
AddS1LibToPath()
# Load the libs
from S1OwnLib import (ReturnRealCalibrationOTBValue,
GetFileByExtensionFromDirectory,
GetNewDatesFromListZipFilesInput,
ReprojVector,
CheckAllDifferentRelativeOrbit,
getS1ZipByTile,
CreateShapeFromPath,
ProcessS1ZipDataset,
ExtractZipFileFromPartFileName,
ExtractFullZipFile,
GenerateTemporalAverage)
'''
NOTES
Things to do
- Optimize the ortorectification by using extent of the polygon to replace clip
'''
##Sentinel-1 IW GRD Batch Processing=group
##1 - Cal. + Orthorect. over Orbits=name
##Input_Data_Folder=folder
##DEM_Folder=folder
##Input_Polygon_File=vector
##Relative_Orbit_Field_Name=field Input_Polygon_File
##Relative_Orbit_To_Process=string 120-47
##Average_All_Date=boolean false
##Calibration_Type=string Gamma0
##Output_EPSG=crs
##Output_Resolution=number 10
##Output_Data_Folder=folder
##Ram=number 512
# END TO DELETE
'''
Input OF THE PROGRAM
'''
'''
This string have to contain the folder path that contain all the unziped S1
data to process
'''
# Input_Data_Folder = '/home/DataStorage/FrenchGuyana/Input'
# Input_Data_Folder = '/home/osboxes/RemoteSensing/PreTraitementsS1/S1GRDZip'
'''
This string have to contain the folder path that contain all dem covering the
study area. Tiff format is required
'''
# DEM_Folder = '/home/DataStorage/FrenchGuyana/SRTM'
# DEM_Folder = '/home/osboxes/RemoteSensing/PreTraitementsS1/DEM'
'''
Path of the vector file that contain one or more polygon to process
Each polygone will be process independantly like independant tile
This vector have to contain one field with integer type containing
the Relative orbit number to process for the current polygon
'''
# Input_Polygon_File = '/home/DataStorage/TestGuyane/FrenchGuyaneStudyAreaOrbit.shp'
# Input_Polygon_File = '/home/osboxes/RemoteSensing/PreTraitementsS1/Vecteur/MyStudyArea.shp'
'''
Name of the field containing the relative orbit
'''
# Relative_Orbit_Field_Name = 'relorb'
'''
Value of all the relative orbit to process. Have to be separated by '-' like
120-47
'''
# Relative_Orbit_To_Process = '47'
'''
String containing the calibration type:
Sigma0
Gamma0
Beta0
'''
# Calibration_Type = 'Gamma0'
'''
Output EPSG (only in meter)
'''
# Output_EPSG = 'EPSG:32622'
# Output_EPSG = 'EPSG:32622'
'''
Output resolution in meter
'''
# Output_Resolution = '10'
'''
Output folder that will contain one subfolder for each processed polygon and
each relative orbit
'''
# Output_Data_Folder = '/home/DataProcessing/FrenchGuyana/Ortho'
# Output_Data_Folder = '/home/osboxes/RemoteSensing/PreTraitementsS1/S1Ortho'
'''
Amount of allocate ram
In case of use half of the available memory (not the physical memory)
'''
# Ram = 500
# Average_All_Date = True
'''
THE PROGRAM ITSELF
'''
# Internal variable
Noise = 'false'
'''
Main step
1 - Transform Calibration user input into OTB command and get its acronym
2 - List ALL Sentinel-1 Manifest file
3 - Filter the one that not already processed in the output folder
4 - Scan path for all polygon and control that there is different path for all
polygon
5 - Loop thru relative orbit given by the user and
5a - Loop thru polygon
For each relative orbit loop thru all the polygon of this orbit
- Create on subfolder per polygon
- Filter all the data to take the one which intersect the study area
- Process the data
'''
# Logging file parameters
TimeNow = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
LogFile = modOsJoinPath([Output_Data_Folder, 'LogFile-OrthoByPath_' + TimeNow + '.txt'])
logging.basicConfig(filename=LogFile,level=logging.INFO)
'''
- List zip files in input and filter comparing output date do get the new one
-
'''
# 1 - Transform Calibration user input into OTB command and get its acronym
OTBCalibrationType, CalibrationName = ReturnRealCalibrationOTBValue(Calibration_Type)
# 2 - List ALL Sentinel-1 Manifest file
# List all SENTINEL-1 manifest.safe files
ZipFiles = GetFileByExtensionFromDirectory(Input_Data_Folder, '.zip')
# 3 - Filter the one that not already processed in the output folder
ZipFiles = GetNewDatesFromListZipFilesInput(ZipFiles, Input_Data_Folder, Output_Data_Folder)
# 4 - Scan path for all polygon and control that there is different path for all
# polygon
# Get Path name list user
PathUserList = Relative_Orbit_To_Process.split('-')
PathUserList = [int(path) for path in PathUserList]
CheckAllDifferentRelativeOrbit(Input_Polygon_File,Relative_Orbit_Field_Name, PathUserList)
# Reproject Shapefile to 4326 to be compliant with S1 raw data
Input_Polygon_FileEPSG4326 = Input_Polygon_File.replace('.shp', 'EPSG4326.shp')
ReprojVector(Input_Polygon_File, Input_Polygon_FileEPSG4326, 4326)
# 5 - Loop thru relative orbit given by the user and
# 5a - Loop thru polygon
for userPath in PathUserList:
# Filter files that intersect the required path
intersectRaster = getS1ZipByTile(Input_Polygon_FileEPSG4326,
ZipFiles,
Relative_Orbit_Field_Name,
userPath)
# If intersect is not empty we create output directory
if len(intersectRaster) > 0:
PathDir = modOsJoinPath([Output_Data_Folder, 'p' + str(userPath)])
if not os.path.exists(PathDir):
os.makedirs(PathDir)
else: # if no data go to next path
continue
# Create Shape file of the current path
PathShape = modOsJoinPath([PathDir, 'p' + str(userPath) + '.shp'])
CreateShapeFromPath(Input_Polygon_FileEPSG4326,
Relative_Orbit_Field_Name,
userPath,
PathShape)
# Run the process
ProcessS1ZipDataset(intersectRaster,
PathDir,
PathShape,
DEM_Folder,
Output_Resolution,
Calibration_Type,
Noise,
CalibrationName,
OTBCalibrationType,
Output_EPSG,
Ram)
# Average all data if enable
if Average_All_Date:
GenerateTemporalAverage(PathDir, True, Ram)
GeomFiles=GetFileByExtensionFromDirectory(Output_Data_Folder, '.geom')
for GFile in GeomFiles:
if os.path.exists(GFile):
os.remove(GFile)
This diff is collapsed.
# QGIS Header
##Sentinel-1 IW GRD Batch Processing=group
##Time Series Statistics=name
##Input_Data_Folder=folder
##Start_Analysis_Date=string 20160124
##End_Analysis_Date=string 20161123
##*ParameterBoolean|Median|Median|True|True
##*ParameterBoolean|Mean|Mean|True|True
##*ParameterBoolean|Mad|Median Absolute Deviation|True|True
##*ParameterBoolean|Stdev|Standard deviation|True|True
##*ParameterBoolean|Min|Minimum|True|True
##*ParameterBoolean|Max|Maximum|True|True
##*ParameterBoolean|P10|10th percentile|True|True
##*ParameterBoolean|P90|90th percentile|True|True
##*ParameterBoolean|P25|25th percentile|True|True
##*ParameterBoolean|P75|75th percentile|True|True
##Output_Data_Folder=folder
##Ram=number 512
# Authors: Cedric Lardeux et Pierre-Louis Frison
StatListToCompute = []
if Median:
StatListToCompute.append('Median')
if Mean:
StatListToCompute.append('Mean')
if Mad:
StatListToCompute.append('Mad')
if Stdev:
StatListToCompute.append('Stdev')
if Min:
StatListToCompute.append('Min')
if Max:
StatListToCompute.append('Max')
if P10:
StatListToCompute.append('P10')
if P90:
StatListToCompute.append('P90')
if P25:
StatListToCompute.append('P25')
if P75:
StatListToCompute.append('P75')
StatListToCompute = '_'.join(StatListToCompute)
# Import LIB
import os, sys, glob, re, subprocess
from qgis.core import QgsApplication
QgisSettingPath = QgsApplication.qgisSettingsDirPath()
ScriptPath = os.path.join(QgisSettingPath, 'processing/scripts/TemporalStats.py')
ScriptPath = ScriptPath.replace("\\","/")
cmd = "python "
cmd += ScriptPath
cmd += ' -InDir ' + Input_Data_Folder
cmd += ' -StartDate ' + str(Start_Analysis_Date)
cmd += ' -EndDate ' + str(End_Analysis_Date)
cmd += ' -StatListToCompute ' + StatListToCompute
cmd += ' -OutDir ' + Output_Data_Folder
cmd += ' -Ram ' + str(Ram)
p1 = subprocess.Popen (cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret= p1.communicate()[1]
\ No newline at end of file
# Work To Do
## Add download script
Have choice to use the folowing server
- Peps
- Vertex ASF
- Sentinel data huh now called Copernicus datahub
## Debug / Improve
- For temporal filtering take into accoun background (-30dB) using mask like here <https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.ma.mean.html>
- - use rios compression in output <https://spectraldifferences.wordpress.com/2014/02/20/set-output-file-type-and-options-in-rios/>
-
## New features
- On temporal filtering update input by giving the main directory conataining the folder pxx pyy that correspong to the relative orbit. So user have to give the path to process
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment