Add initial files.
This commit is contained in:
jmsgrogan
parent
1f228d3d2a
commit
00fa3da2eb
24 changed files with 14465 additions and 0 deletions
0
format_conversion/__init__.py
Normal file
0
format_conversion/__init__.py
Normal file
50
format_conversion/bftools_wrapper.py
Normal file
50
format_conversion/bftools_wrapper.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
"""
|
||||
Wrapper over bftools
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
def convert(input_path, output_path, tile_x=None, tile_y=None,
|
||||
channel=0, series=0, crop_indices = None, bigtiff=False):
|
||||
|
||||
"""
|
||||
Convert an input image in ZEISS LSM or CZI format to an OME TIFF.
|
||||
Slice spacing information is lost in this operation for some reason.
|
||||
"""
|
||||
|
||||
# Set up the command for bfconvert
|
||||
bf_tools_dir = os.getenv('BFTOOLS_DIR', os.getcwd()) + "/"
|
||||
command = bf_tools_dir + "/bfconvert "
|
||||
if not os.path.exists(output_path):
|
||||
os.makedirs(output_path)
|
||||
output_path += "/" + os.path.splitext(input_path)[0] + "_T%t"
|
||||
|
||||
# Only one channel at a time
|
||||
command += " -channel " + str(channel) + " "
|
||||
output_path += "_C%c"
|
||||
# Only one series at a time
|
||||
command += " -series " + str(series) + " "
|
||||
output_path += "_S%s"
|
||||
|
||||
# Set up tiles
|
||||
if tile_x is not None and tile_y is not None:
|
||||
command += " -tilex " + str(tile_x) + " -tiley " + str(tile_y) + " "
|
||||
output_path += "_tile_X%x_Y%y_ID_%m"
|
||||
|
||||
|
||||
# bigtiff
|
||||
if bigtiff:
|
||||
command += " -bigtiff "
|
||||
|
||||
# crop
|
||||
if crop_indices is not None:
|
||||
command += " -crop " + str(int(crop_indices[4])) + "," + str(int(crop_indices[5]))
|
||||
command += "," + str(int(crop_indices[6])) + "," + str(int(crop_indices[7])) + " "
|
||||
global_index = crop_indices[0] + crop_indices[1]*crop_indices[2]
|
||||
output_path += "_x_"+ str(int(crop_indices[0])) + "_y_" + str(int(crop_indices[1])) + "_m_" + str(int(global_index))
|
||||
command += input_path + " " + output_path + ".tiff"
|
||||
|
||||
# Do the conversion
|
||||
p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
|
||||
p.wait()
|
||||
103
format_conversion/extract_zeiss_metadata.py
Normal file
103
format_conversion/extract_zeiss_metadata.py
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
#!/usr/bin/env python
|
||||
"""
|
||||
Extract and Parse Metadata from IMS/LSM/CZI files.
|
||||
"""
|
||||
|
||||
from argparse import ArgumentParser
|
||||
import os
|
||||
import subprocess
|
||||
import logging
|
||||
import utility
|
||||
import ome_schema
|
||||
|
||||
def extract_metadata(input_path, output_path):
|
||||
|
||||
"""
|
||||
Extract OME metadata from the input file and write it out as a nicely formatted xml using
|
||||
bftools. (http://www.openmicroscopy.org/site/support/bio-formats5.3/users/comlinetools/display.html)
|
||||
"""
|
||||
|
||||
bf_tools_dir = os.getenv('BFTOOLS_DIR', os.getcwd()) + "/"
|
||||
command = bf_tools_dir +"showinf -omexml-only -nopix " + input_path + " | " + bf_tools_dir + "xmlindent > " + output_path
|
||||
p = subprocess.Popen(command, shell=True)
|
||||
p.wait()
|
||||
|
||||
def get_metadata_as_class(input_xml_path):
|
||||
|
||||
"""
|
||||
Return the OME metadata from the input XML file as a Python class. The class is automatically generated
|
||||
using pyxbgen (http://pyxb.sourceforge.net/pyxbgen_cli.html) and the current OME XML Schema
|
||||
(https://www.openmicroscopy.org/Schemas/OME/2016-06/ome.xsd).
|
||||
|
||||
If you need to use a newer schema you need to regenerate the file ome_schema.py by doing:
|
||||
pip install pyxb
|
||||
pyxbgen -m ome_schema -u https://www.openmicroscopy.org/Schemas/OME/2016-06/ome.xsd
|
||||
|
||||
where the web address points to the new schema. You can then access the elements of the OME XML as
|
||||
instance attributes etc.
|
||||
"""
|
||||
|
||||
xml = open(input_xml_path).read()
|
||||
image_metadata = ome_schema.CreateFromDocument(xml)
|
||||
return image_metadata
|
||||
|
||||
def integer_color_to_rgb(color):
|
||||
|
||||
"""
|
||||
Convert integer color to (r,g,b)
|
||||
"""
|
||||
|
||||
return ((color >> 16) & 255, (color >> 8) & 255, color & 255)
|
||||
|
||||
|
||||
def print_metadata_overview(image_metadata):
|
||||
|
||||
"""
|
||||
Print a reader-friendly metadata summary
|
||||
"""
|
||||
|
||||
print "Number of Images: ", len(image_metadata.Image)
|
||||
print "Image '0' - Name: ", image_metadata.Image[0].Name
|
||||
print "Image '0' - Num Channels: ", image_metadata.Image[0].Pixels.SizeC
|
||||
print "Image '0' - Num Times: ", image_metadata.Image[0].Pixels.SizeT
|
||||
pixel_size_x = image_metadata.Image[0].Pixels.PhysicalSizeX
|
||||
pixel_size_y = image_metadata.Image[0].Pixels.PhysicalSizeY
|
||||
pixel_size_z = image_metadata.Image[0].Pixels.PhysicalSizeZ
|
||||
pixel_unit_x = image_metadata.Image[0].Pixels.PhysicalSizeXUnit
|
||||
pixel_unit_y = image_metadata.Image[0].Pixels.PhysicalSizeYUnit
|
||||
pixel_unit_z = image_metadata.Image[0].Pixels.PhysicalSizeZUnit
|
||||
print "Image '0' - Pixel Physical Size X: ", pixel_size_x, pixel_unit_x
|
||||
print "Image '0' - Pixel Physical Size Y: ", pixel_size_y, pixel_unit_y
|
||||
print "Image '0' - Pixel Physical Size Z: ", pixel_size_z, pixel_unit_z
|
||||
print "Image '0' - Pixel Size X: ", image_metadata.Image[0].Pixels.SizeX
|
||||
print "Image '0' - Pixel Size Y:", image_metadata.Image[0].Pixels.SizeY
|
||||
print "Image '0' - Pixel Size Z:", image_metadata.Image[0].Pixels.SizeZ
|
||||
print "Image '0' - Pixel Dimension Order: ", image_metadata.Image[0].Pixels.DimensionOrder
|
||||
print "Image '0' - Pixel Bits: ", image_metadata.Image[0].Pixels.SignificantBits
|
||||
for idx, eachChannel in enumerate(image_metadata.Image[0].Pixels.Channel):
|
||||
print "Image '0' - Channel " +str(idx) + " Color: ", integer_color_to_rgb(eachChannel.Color)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# Do setup
|
||||
tool_name = "extract_metadata"
|
||||
utility.do_setup(tool_name)
|
||||
logger1 = logging.getLogger('format_conversion.'+tool_name)
|
||||
|
||||
# Suppress XML Parse warnings
|
||||
pyxb_logger = logging.getLogger('pyxb')
|
||||
pyxb_logger.setLevel(logging.CRITICAL)
|
||||
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("-i", "--input_file", type=str, help='Input file in a ZEISS format.')
|
||||
parser.add_argument("-o", "--output_file", type=str, help='Output metadata file.')
|
||||
parser.add_argument("--verbose", type=bool, help='Output a simple metadata summary.')
|
||||
args = parser.parse_args()
|
||||
|
||||
logger1.info('Reading Metadata At: ' + args.input_file)
|
||||
extract_metadata(args.input_file, args.output_file)
|
||||
|
||||
if(args.verbose):
|
||||
image_metadata = get_metadata_as_class(args.output_file)
|
||||
print_metadata_overview(image_metadata)
|
||||
logger1.info('Completed Reading Metadata')
|
||||
12562
format_conversion/ome_schema.py
Normal file
12562
format_conversion/ome_schema.py
Normal file
File diff suppressed because it is too large
Load diff
120
format_conversion/tiff_to_vtk.py
Normal file
120
format_conversion/tiff_to_vtk.py
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
"""
|
||||
Convert to VTK. Use pvti files for final assembly.
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
import math
|
||||
from argparse import ArgumentParser
|
||||
from functools import partial
|
||||
import multiprocessing as mp
|
||||
import vtk
|
||||
import utility
|
||||
import extract_zeiss_metadata
|
||||
|
||||
|
||||
def do_conversion(file_attributes, working_directory):
|
||||
|
||||
reader = vtk.vtkTIFFReader()
|
||||
reader.SetFileName(file_attributes["FileName"])
|
||||
reader.Update()
|
||||
|
||||
image_data = reader.GetOutput()
|
||||
image_data.SetExtent(file_attributes["Extents"])
|
||||
image_data.SetSpacing(file_attributes["PhysicalSize"])
|
||||
|
||||
out_path = working_directory + "/" + os.path.splitext(os.path.basename(file_attributes["FileName"]))[0] + ".vti"
|
||||
writer = vtk.vtkXMLImageDataWriter()
|
||||
writer.SetInputData(image_data)
|
||||
writer.SetFileName(out_path)
|
||||
writer.Write()
|
||||
|
||||
def convert_to_vti(input_folder, metadata_file, tile_dimensions, downsample_dimensions):
|
||||
|
||||
extract_zeiss_metadata.extract_metadata(metadata_file, os.getcwd() + "temp_metadata.xml")
|
||||
image_metadata = extract_zeiss_metadata.get_metadata_as_class(os.getcwd() + "temp_metadata.xml")
|
||||
|
||||
num_x = int(math.floor(image_metadata.Image[0].Pixels.SizeX/tile_dimensions[0]))+1
|
||||
num_y = int(math.floor(image_metadata.Image[0].Pixels.SizeY/tile_dimensions[1]))+1
|
||||
size_x = int(math.floor(tile_dimensions[0]/downsample_dimensions[0]))+1
|
||||
size_y = int(math.floor(tile_dimensions[1]/downsample_dimensions[1]))+1
|
||||
size_z = int(math.floor(image_metadata.Image[0].Pixels.SizeZ/downsample_dimensions[2]))
|
||||
physical_x = float(image_metadata.Image[0].Pixels.PhysicalSizeX)*float(downsample_dimensions[0])
|
||||
physical_y = float(image_metadata.Image[0].Pixels.PhysicalSizeY)*float(downsample_dimensions[1])
|
||||
physical_z = float(image_metadata.Image[0].Pixels.PhysicalSizeZ)*float(downsample_dimensions[2])
|
||||
|
||||
# Grab the file base name. Assume the input folder only has the tiff files we want to convert
|
||||
path_base_name = ""
|
||||
for eachFile in os.listdir(input_folder):
|
||||
if eachFile.endswith(".tiff"):
|
||||
loc_start_index = eachFile.find("_tile")
|
||||
if loc_start_index>-1:
|
||||
path_base_name = input_folder + "/" + eachFile[:loc_start_index]
|
||||
break
|
||||
|
||||
file_attribute_collection = []
|
||||
for jdx in range(num_y):
|
||||
for idx in range(num_x):
|
||||
global_index = idx + jdx*num_x
|
||||
file_name = path_base_name + "_tile_X" + str(idx) + "_Y" + str(jdx) + "_ID_" + str(global_index)+"_pixels_only_downsampled.tiff"
|
||||
min_ext_x = idx*size_x
|
||||
min_ext_y = jdx*size_y
|
||||
max_ext_x = (idx+1)*size_x-1
|
||||
max_ext_y = (jdx+1)*size_y-1
|
||||
min_ext_z = 0
|
||||
max_ext_z = size_z-1
|
||||
if idx == num_x - 1:
|
||||
max_ext_x = int(math.floor(image_metadata.Image[0].Pixels.SizeX/downsample_dimensions[0]))+2
|
||||
if jdx == num_y - 1:
|
||||
max_ext_y = int(math.floor(image_metadata.Image[0].Pixels.SizeY/downsample_dimensions[1]))+2
|
||||
extents = [min_ext_x, max_ext_x, min_ext_y, max_ext_y, min_ext_z, max_ext_z]
|
||||
file_attributes = {}
|
||||
file_attributes["PhysicalSize"]= (physical_x, physical_y, physical_z)
|
||||
file_attributes["Extents"]= extents
|
||||
file_attributes["FileName"]= file_name
|
||||
file_attribute_collection.append(file_attributes)
|
||||
|
||||
# Make a partial function for multiple arg mapping
|
||||
working_directory = os.getcwd() + "/VTK_Files/"
|
||||
if not os.path.exists(working_directory):
|
||||
os.makedirs(working_directory)
|
||||
partial_extract_pixel_values_only = partial(do_conversion, working_directory=working_directory)
|
||||
|
||||
# Limit to max 6 cpus
|
||||
num_available_cpus = mp.cpu_count()
|
||||
if num_available_cpus > 6:
|
||||
num_available_cpus = 6
|
||||
|
||||
# Distribute across processes - Be conservative with available CPUs
|
||||
run_cpus = int(math.floor(num_available_cpus/2))
|
||||
if run_cpus == 0:
|
||||
run_cpus = 1
|
||||
#pool = mp.Pool(processes = run_cpus)
|
||||
pool = mp.Pool(processes = 1)
|
||||
pool.map(partial_extract_pixel_values_only, file_attribute_collection)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# Do setup
|
||||
tool_name = "zeiss_to_tiff"
|
||||
utility.do_setup(tool_name)
|
||||
logger1 = logging.getLogger('tiff_to_vtk.'+tool_name)
|
||||
|
||||
# Suppress XML Parse warnings
|
||||
pyxb_logger = logging.getLogger('pyxb')
|
||||
pyxb_logger.setLevel(logging.CRITICAL)
|
||||
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("-i", "--input_folder", type=str, help='Folder with tiffs to be converted.')
|
||||
parser.add_argument("-m", "--metadatafile", type=str, help='File with image metadata.')
|
||||
parser.add_argument("-tx", "--tile_x", type=int, help='X Tile size.')
|
||||
parser.add_argument("-ty", "--tile_y", type=int, help='Y Tile Size.')
|
||||
parser.add_argument("-dx", "--downsample_x", type=int, help='X Downsample Factor.', default=1)
|
||||
parser.add_argument("-dy", "--downsample_y", type=int, help='Y Downsample Factor.', default=1)
|
||||
parser.add_argument("-dz", "--downsample_z", type=int, help='Y Downsample Factor.', default=1)
|
||||
args = parser.parse_args()
|
||||
|
||||
logger1.info('Start Converting Images to VTI at: ' + args.input_folder)
|
||||
convert_to_vti(args.input_folder, args.metadatafile, (args.tile_x, args.tile_y),
|
||||
(args.downsample_x, args.downsample_y, args.downsample_z))
|
||||
logger1.info('Completed Converting')
|
||||
97
format_conversion/zeiss_to_tiff.py
Normal file
97
format_conversion/zeiss_to_tiff.py
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
"""
|
||||
Conversion of IMS/LSM/CZI files to OME and then Generic TIFFs
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
import math
|
||||
from argparse import ArgumentParser
|
||||
from functools import partial
|
||||
import multiprocessing as mp
|
||||
import skimage.external.tifffile
|
||||
import utility
|
||||
import extract_zeiss_metadata
|
||||
import bftools_wrapper
|
||||
|
||||
def extract_pixel_values_only(input_file, working_directory, channel=0, significant_bits=8):
|
||||
|
||||
im = skimage.external.tifffile.imread(input_file)
|
||||
pixel_vals_only = im[0, :, 0, channel, :, :] # Strip excess data
|
||||
out_path = working_directory + "/" + os.path.splitext(os.path.basename(input_file))[0] + "_pixels_only.tiff"
|
||||
|
||||
if significant_bits==8:
|
||||
skimage.external.tifffile.imsave(out_path, skimage.img_as_ubyte(pixel_vals_only))
|
||||
elif significant_bits==16:
|
||||
skimage.external.tifffile.imsave(out_path, skimage.img_as_uint(pixel_vals_only))
|
||||
else:
|
||||
# Let skiimage decide
|
||||
skimage.external.tifffile.imsave(out_path, pixel_vals_only)
|
||||
|
||||
def do_conversion(input_file, output_file, channel = 0, series = 0):
|
||||
#
|
||||
# # Get the image metadata
|
||||
extract_zeiss_metadata.extract_metadata(input_file, os.getcwd() + "temp_metadata.xml")
|
||||
image_metadata = extract_zeiss_metadata.get_metadata_as_class(os.getcwd() + "temp_metadata.xml")
|
||||
#
|
||||
# # Choose the tile size
|
||||
tile_x = 1024
|
||||
tile_y = 1024
|
||||
|
||||
# Black box JVM conversion via CLI. Be skeptical of outputs,
|
||||
# at least the Z-physical size is silently lost!!!
|
||||
bftools_wrapper.convert(input_file, output_file, tile_x, tile_y, channel, series)
|
||||
|
||||
# Make versions with just XYZ pixel values for easier use in scikit-image/VTK etc.
|
||||
# DANGER, now all physical size is lost. Only trust physical sizes
|
||||
# in the standalone xml metadata file, obtained (and validated) using `extract_zeiss_metadata()`.
|
||||
|
||||
# Collect the file names
|
||||
file_names = []
|
||||
for eachFile in os.listdir(output_file):
|
||||
if eachFile.endswith(".tiff"):
|
||||
file_names.append(output_file + "/" + eachFile)
|
||||
|
||||
# Make a partial function for multiple arg mapping
|
||||
working_directory = output_file + "/Pixel_Values_Only/"
|
||||
if not os.path.exists(working_directory):
|
||||
os.makedirs(working_directory)
|
||||
|
||||
significant_bits = int(image_metadata.Image[0].Pixels.SignificantBits)
|
||||
partial_extract_pixel_values_only = partial(extract_pixel_values_only,
|
||||
working_directory=working_directory,
|
||||
channel=channel,
|
||||
significant_bits=significant_bits)
|
||||
|
||||
# Limit to max 6 cpus
|
||||
num_available_cpus = mp.cpu_count()
|
||||
if num_available_cpus > 6:
|
||||
num_available_cpus = 6
|
||||
|
||||
# Distribute across processes - Be conservative with available CPUs
|
||||
run_cpus = int(math.floor(num_available_cpus/2))
|
||||
if run_cpus == 0:
|
||||
run_cpus = 1
|
||||
run_cpus = 1
|
||||
#pool = mp.Pool(processes = run_cpus)
|
||||
pool = mp.Pool(processes = 1)
|
||||
pool.map(partial_extract_pixel_values_only, file_names)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# Do setup
|
||||
tool_name = "zeiss_to_tiff"
|
||||
utility.do_setup(tool_name)
|
||||
logger1 = logging.getLogger('format_conversion.'+tool_name)
|
||||
|
||||
# Suppress XML Parse warnings
|
||||
pyxb_logger = logging.getLogger('pyxb')
|
||||
pyxb_logger.setLevel(logging.CRITICAL)
|
||||
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument("-i", "--input_file", type=str, help='Input file in a ZEISS format.')
|
||||
parser.add_argument("-o", "--output_folder", type=str, help='Output Tiff Location.')
|
||||
args = parser.parse_args()
|
||||
|
||||
logger1.info('Start Converting Image at: ' + args.input_file)
|
||||
do_conversion(args.input_file, args.output_folder, channel = 1, series = 0)
|
||||
logger1.info('Completed Processing')
|
||||
Loading…
Add table
Add a link
Reference in a new issue