Commit a1c0f310 authored by Nick R. Papior's avatar Nick R. Papior
Browse files

Enabled BLAS and LAPACK sources shipped with siesta fixes lp:1593706

- Updated blas.f and lapack.f to encompass all required
  routines to succesfully compile siesta without having the
  blas and lapack libraries.

  We still do not recommend this as performance libraries
  (especially BLAS) will drastically improve performance.

  The current sources (without comments) are taken from
  the lapack svn repository at revision 1776 (v3.6.1)

- To make it easier to update the scripts in future
  commits a create.sh script and linalg2file.py script
  are added which enables to retrieve the correct sources
  from the basic lapack/blas sources.

- Updated documentation which describes libsiestaBLAS/LAPACK.a

- Enabled libsiestaBLAS/LAPACK.a in tbtrans

  
parent ba1b604e
......@@ -519,7 +519,11 @@ libraries and how each of them may be added to the compilation step
\begin{itemize}
\item If you use your *nix distribution package manager to install
BLAS you are bound to have a poor performance. Please try and use
performance libraries.
performance libraries, whenever possible!
\item If you do not have the BLAS library you may use the BLAS
library shipped with \siesta. To do so simply add
\shell{libsiestaBLAS.a} to the \shell{COMP_LIBS} variable.
\end{itemize}
To add BLAS to the \shell{arch.make} file you need to add the
......@@ -552,6 +556,10 @@ libraries and how each of them may be added to the compilation step
enables the inclusion of the LAPACK routines. This is advised.}
or MKL library from Intel)
If you do not have the LAPACK library you may use the LAPACK
library shipped with \siesta. To do so simply add
\shell{libsiestaLAPACK.a} to the \shell{COMP_LIBS} variable.
Example variables
\begin{shellexample}
# OpenBLAS (OpenBLAS will default to build in LAPACK 3.6)
......
......@@ -8,7 +8,7 @@ For the LAPACK code, the conditions in the file LICENSE apply.
Use the code in this directory if you do not have compiled (presumably
optimized) blas and lapack libraries in your system.
Define COMP_LIBS = linalg.a
Define COMP_LIBS = libsiestaLAPACK.a libsiestaBLAS.a
in your XXXXX.make file
......@@ -21,4 +21,4 @@ from version 3.0 of Lapack. In that case, define
You might already have an optimized blas, but not lapack, in which case:
COMP_LIBS= liblapack.a $BLAS_LIBS
COMP_LIBS= libsiestaLAPACK.a $BLAS_LIBS
This diff is collapsed.
# This file contains all files that should be
# manually added (not detected by linalg2file)
# These are needed for scalapack
zsymm.f
zsyrk.f
zsyr2k.f
cdotc.f
cdotu.f
dasum.f
dcabs1.f
ddot.f
dnrm2.f
dsdot.f
dzasum.f
dznrm2.f
icamax.f
idamax.f
isamax.f
izamax.f
lsame.f
sasum.f
scabs1.f
scasum.f
scnrm2.f
sdot.f
sdsdot.f
snrm2.f
zdotc.f
zdotu.f
#!/bin/bash
cur_dir=$(pwd)
# First gather the lapack sources
python linalg2file.py -d $cur_dir/.. $cur_dir/../../Util/TS/TBtrans \
-l ~/LA/lapack/SRC ~/LA/lapack/SRC/DEPRECATED ~/LA/lapack/INSTALL \
--list-add-file lapack_add.files \
--list-add-routine lapack_add.routines \
--list-remove-file lapack_remove.files \
--list-remove-routine lapack_remove.routines \
-o lapack.f
# Now we need to locate the lapack sources as well
python linalg2file.py -d $cur_dir/.. $cur_dir/../../Util/TS/TBtrans \
-f lapack.f \
-l ~/LA/lapack/BLAS/SRC \
--list-add-file blas_add.files -o blas.f
This diff is collapsed.
# This file contains all files that should be
# manually added (not detected by linalg2file)
# This file is in INSTALL
slamch.f
dlamch.f
# Generic functions
chla_transtype.f
cladiv.f
cla_gbrcond_c.f
cla_gbrcond_x.f
cla_gbrpvgrw.f
cla_gercond_c.f
cla_gercond_x.f
cla_gerpvgrw.f
cla_hercond_c.f
cla_hercond_x.f
cla_herpvgrw.f
clangb.f
clange.f
clangt.f
clanhb.f
clanhe.f
clanhf.f
clanhp.f
clanhs.f
clanht.f
clansb.f
clansp.f
clansy.f
clantb.f
clantp.f
clantr.f
cla_porcond_c.f
cla_porcond_x.f
cla_porpvgrw.f
cla_syrcond_c.f
cla_syrcond_x.f
cla_syrpvgrw.f
disnan.f
dladiv.f
dla_gbrcond.f
dla_gbrpvgrw.f
dla_gercond.f
dla_gerpvgrw.f
dlaisnan.f
dlaneg.f
dlangb.f
dlange.f
dlangt.f
dlanhs.f
dlansb.f
dlansf.f
dlansp.f
dlanst.f
dlansy.f
dlantb.f
dlantp.f
dlantr.f
dla_porcond.f
dla_porpvgrw.f
dlapy2.f
dlapy3.f
dla_syrcond.f
dla_syrpvgrw.f
dzsum1.f
icmax1.f
ieeeck.f
ilaclc.f
ilaclr.f
iladiag.f
iladlc.f
iladlr.f
ilaenv.f
ilaprec.f
ilaslc.f
ilaslr.f
ilatrans.f
ilauplo.f
ilazlc.f
ilazlr.f
iparmq.f
izmax1.f
lsamen.f
scsum1.f
sisnan.f
sladiv.f
sla_gbrcond.f
sla_gbrpvgrw.f
sla_gercond.f
sla_gerpvgrw.f
slaisnan.f
slaneg.f
slangb.f
slange.f
slangt.f
slanhs.f
slansb.f
slansf.f
slansp.f
slanst.f
slansy.f
slantb.f
slantp.f
slantr.f
sla_porcond.f
sla_porpvgrw.f
slapy2.f
slapy3.f
sla_syrcond.f
sla_syrpvgrw.f
zladiv.f
zla_gbrcond_c.f
zla_gbrcond_x.f
zla_gbrpvgrw.f
zla_gercond_c.f
zla_gercond_x.f
zla_gerpvgrw.f
zla_hercond_c.f
zla_hercond_x.f
zla_herpvgrw.f
zlangb.f
zlange.f
zlangt.f
zlanhb.f
zlanhe.f
zlanhf.f
zlanhp.f
zlanhs.f
zlanht.f
zlansb.f
zlansp.f
zlansy.f
zlantb.f
zlantp.f
zlantr.f
zla_porcond_c.f
zla_porcond_x.f
zla_porpvgrw.f
zla_syrcond_c.f
zla_syrcond_x.f
zla_syrpvgrw.f
dlarra
dlarrb
dlarrc
dlarrd
dlarrk
dlarrv
dlamchf77.f
dlamchtst.f
lsame.f
slamchf77.f
slamchtst.f
# This file contains all subroutines that should be
# manually removed
lsame
xerbla
#!/usr/bin/env python
from __future__ import print_function
# Collect all routines into a single file
# This is currently used for constructing the
# resulting combined libraries for siesta
# with LA
import argparse
import sys, os, re
import os.path as osp
import glob
def reduce_sort(lst):
lst = list(set(lst))
lst.sort()
return lst
def list_read(files):
lines = []
for f in files:
tmp = open(f).readlines()
for line in tmp:
line = line.strip()
if len(line) == 0: continue
line = line.lower().replace('\n','')
if not line.startswith('#'):
lines.append(line.replace('\r',''))
return lines
def strip_elements(lst, els):
if els is None:
return lst
for el in els:
try:
lst.remove(el)
except: pass
return lst
# Luckily all files are in Fortran 77 format
# So it is easy to find subroutines used.
def calls_file(f):
""" Reads file `f` and returns all subroutine calls from this file """
# Only parse if it is a file
if not osp.isfile(f): return []
# Easy check for format
is_f77 = f[-1] == 'f' or f[-1] == 'F'
calls = []
with open(f, 'r') as fh:
for line in fh:
# Remove all regular comments.
line = line.lower().split('!')[0].strip()
try:
idx = line.index('call ')
except:
continue
# We have a call in the line...
call = line[idx+5:].split('(')[0]
calls.append(call.strip())
return calls
def calls_files(files):
""" Reads all sources F, F90, f, f90 and returns a list of all calls
made in all those files.
"""
calls = []
for f in files:
calls.extend(calls_file(f))
return calls
def calls_directory(d):
""" Reads all sources F, F90, f, f90 and returns a list of all calls
made in all those files.
"""
# Only parse if it is a directory
if not osp.isdir(d): return []
calls = []
for end in ['F', 'F90', 'f', 'f90']:
files = glob.glob(osp.join(d,'*.'+end))
calls.extend(calls_files(files))
return calls
def subroutines_file(f):
""" Reads file `f` and returns all subroutine definitions from this file """
# Only parse if it is a file
if not osp.isfile(f): return []
subs = []
with open(f, 'r') as fh:
for line in fh:
# Remove all regular comments.
line = line.lower().split('!')[0].strip()
try:
idx = line.index('subroutine ')
l = 10
except:
continue
# We have a call in the line...
try:
sub = line[idx+l:].split('(')[0]
except:
continue
subs.append(sub.strip())
return subs
def find_files_calls(d, calls):
""" Find files in `d` where the subroutines coinciding with those in `calls` may be found """
# Only parse if it is a directory
if not osp.isdir(d): return []
files = []
for end in ['F', 'F90', 'f', 'f90']:
for f in glob.glob(osp.join(d,'*.'+end)):
subs = subroutines_file(f)
for sub in subs:
if sub in calls:
files.append(f)
return files
def find_files(d, infiles):
""" Find files in `d` where the subroutines coinciding with those in `calls` may be found """
# Only parse if it is a directory
if not osp.isdir(d): return []
files = []
for infile in infiles:
for f in glob.glob(osp.join(d,infile)):
if osp.isfile(f):
files.append(f)
return files
def write_file(f, out, comments=False):
with open(out, 'a') as oh:
oh.write('! SOURCE-FILE = {}\n'.format(f))
with open(f, 'r') as fh:
for line in fh:
if comments:
oh.write(line)
elif line[0] == ' ':
oh.write(line)
def write_files(files, out, comments=False):
for f in files:
write_file(f, out, comments)
if __name__ == '__main__':
# First argument is the directory of the sources
parser = argparse.ArgumentParser(description='Gather BLAS/LAPACK sources from external source directories.')
# Add common options
parser.add_argument('--directory','-d', dest="source_dir", type=str, nargs='+',
help='Specify directory of the sources where the entire source tree is found.')
parser.add_argument('--file','-f', dest="source_file", type=str, nargs='+', default=[],
help='Specify a specific file of sources.')
parser.add_argument('--library-directory','-l', dest="library_dir", type=str, nargs='+',
help='Specify directory of the sources where the entire source tree of the library is found.')
parser.add_argument('--list-add-routine', dest='add_routine', type=str, nargs='+', default=[],
help='Read file with list of routines that are added as calls.')
parser.add_argument('--list-remove-routine', dest='remove_routine', type=str, nargs='+', default=[],
help='Read file with list of routines that are removed as calls.')
parser.add_argument('--list-add-file', dest='add_file', type=str, nargs='+', default=[],
help='Ensure these files are added to the source (found in [library_dir]).')
parser.add_argument('--list-remove-file', dest='remove_file', type=str, nargs='+', default=[],
help='Ensure these files are removed from the source (found in [library_dir]).')
parser.add_argument('--out','-o', dest='out_file', type=str,
help='Output file of resulting used files')
parser.add_argument('--comments', action='store_true', default=False,
help='Strips all comments from the source files')
args = parser.parse_args()
# First remote out file
try:
os.remove(args.out_file)
except: pass
# Read contents of files on command-line
add_routines = list_read(args.add_routine)
rem_routines = list_read(args.remove_routine)
add_files = list_read(args.add_file)
rem_files = list_read(args.remove_file)
# Retrieve all calls in the SOURCE directory
calls = []
for source_dir in args.source_dir:
print('Parsing calls in: {}'.format(source_dir))
calls.extend(calls_directory(source_dir))
# Add custom files
calls.extend(calls_files(args.source_file))
# Add user-defined calls
calls.extend(add_routines)
calls = reduce_sort(calls)
# Strip user-defined subroutines
calls = strip_elements(calls, rem_routines)
# Show how many different routines we actually found
print('Found {} calls in the source directories...\n'.format(len(calls)))
files = []
for library_dir in args.library_dir:
print('Parsing subroutines in: {}'.format(library_dir))
files.extend(find_files_calls(library_dir, calls))
# Now we have all higher level files directly
# called from the source directories
files = reduce_sort(files)
len_source_files = len(files)
# Add custom files
tmp = []
for library_dir in args.library_dir:
files.extend(find_files(library_dir, add_files))
tmp.extend(find_files(library_dir, rem_files))
rem_files = reduce_sort(tmp)
# Now keep finding routines untill all are found
old_len = 0
new_len = 1
while old_len != new_len:
old_len = len(files)
# Be sure to have all sub-set files as well
calls = calls_files(files)
calls = reduce_sort(calls)
# Strip user-defined calls
calls = strip_elements(calls, rem_routines)
print('Found {} calls in the library sources!'.format(len(calls)))
for library_dir in args.library_dir:
files.extend(find_files_calls(library_dir, calls))
# Now we have all higher level files directly
# called from the source directories
# AND we have the library files directory calls
files = reduce_sort(files)
# Remove files
for f in rem_files:
try:
files.remove(f)
except:
pass
new_len = len(files)
print('Require {} / {} files / + lib-files!'.format(len_source_files, len(files)))
write_files(files, args.out_file, comments=args.comments)
......@@ -4,63 +4,94 @@
# GNU General Public License: see COPYING in the top directory
# or http://www.gnu.org/copyleft/gpl.txt.
# See Docs/Contributors.txt for a list of contributors.
default: libsiestaLAPACK.a libsiestaBLAS.a
ARCH_MAKE?=../arch.make
include $(ARCH_MAKE)
###########
# These are now the preferred complete BLAS and LAPACK
# sources.
# They are extracted using the linalg2file.py script.
#
# See create.sh for how the lapack.f and blas.f files are created.
#
# Makefile for Libraries
# Some systems only need the stuff in dc_lapack.f (Divide and Conquer)
# TranSiesta needs zgesv_lapack.f
#
default: linalg.a
# Note that the python script is extremely basic, but functions well for
# F77 code.
libsiestaBLAS.a: blas.o
$(AR) $(ARFLAGS_EXTRA) cru $@ $<
$(RANLIB) $@
libsiestaLAPACK.a: lapack.o
$(AR) $(ARFLAGS_EXTRA) cru $@ $<
$(RANLIB) $@
########################
#
ARCH_MAKE?=../arch.make
include $(ARCH_MAKE)
# Below are old (pre 4.1) temporary libraries for
# linear algebra.
# They should probably not be used as it is better
# to use a consistent library with all functionality.
#
ALL_OBJS= lapack.o dc_lapack.o zgesv_lapack.o zgeev_lapack.o\
svd_omm_lapack.o blas.o machine.o
# We advice on using the newer
# libsiestaLAPACK.a
# and/or
# libsiestaBLAS.a
#
########################
ALL_OBJS = dc_lapack.o zgesv_lapack.o zgeev_lapack.o \
svd_omm_lapack.o old_lapack.o old_blas.o machine.o
# Add machine.o here only if needed...
DC_OBJS= dc_lapack.o # machine.o
DC_OBJS = dc_lapack.o # machine.o
# Used in TranSIESTA
ZGESV_OBJS= zgesv_lapack.o # machine.o
ZGESV_OBJS = zgesv_lapack.o # machine.o
# Used in TBTrans
ZGEEV_OBJS= zgeev_lapack.o
#
ESSL_OBJS= essl_lapack.o
#
default: linalg.a
#