summaryrefslogtreecommitdiff
path: root/scripts/lib
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/lib')
-rw-r--r--scripts/lib/argparse_oe.py169
-rw-r--r--scripts/lib/build_perf/__init__.py31
-rw-r--r--scripts/lib/build_perf/html.py19
-rw-r--r--scripts/lib/build_perf/html/measurement_chart.html50
-rw-r--r--scripts/lib/build_perf/html/report.html206
-rw-r--r--scripts/lib/build_perf/report.py342
-rw-r--r--scripts/lib/build_perf/scrape-html-report.js56
-rw-r--r--scripts/lib/compatlayer/__init__.py308
-rw-r--r--scripts/lib/compatlayer/case.py7
-rw-r--r--scripts/lib/compatlayer/cases/__init__.py (renamed from scripts/lib/wic/3rdparty/pykickstart/__init__.py)0
-rw-r--r--scripts/lib/compatlayer/cases/bsp.py204
-rw-r--r--scripts/lib/compatlayer/cases/common.py91
-rw-r--r--scripts/lib/compatlayer/cases/distro.py26
-rw-r--r--scripts/lib/compatlayer/context.py14
-rw-r--r--scripts/lib/devtool/__init__.py236
-rw-r--r--scripts/lib/devtool/build.py86
-rw-r--r--scripts/lib/devtool/build_image.py174
-rw-r--r--scripts/lib/devtool/build_sdk.py65
-rw-r--r--scripts/lib/devtool/deploy.py323
-rw-r--r--scripts/lib/devtool/package.py60
-rw-r--r--scripts/lib/devtool/runqemu.py74
-rw-r--r--scripts/lib/devtool/sdk.py336
-rw-r--r--scripts/lib/devtool/search.py88
-rw-r--r--scripts/lib/devtool/standard.py2150
-rw-r--r--scripts/lib/devtool/upgrade.py409
-rw-r--r--scripts/lib/devtool/utilcmds.py233
-rw-r--r--scripts/lib/image/__init__.py22
-rw-r--r--scripts/lib/image/canned-wks/mkgummidisk.wks11
-rw-r--r--scripts/lib/image/config/wic.conf6
-rw-r--r--scripts/lib/image/engine.py279
-rw-r--r--scripts/lib/recipetool/append.py457
-rw-r--r--scripts/lib/recipetool/create.py1061
-rw-r--r--scripts/lib/recipetool/create_buildsys.py820
-rw-r--r--scripts/lib/recipetool/create_buildsys_python.py109
-rw-r--r--scripts/lib/recipetool/create_kernel.py99
-rw-r--r--scripts/lib/recipetool/create_kmod.py152
-rw-r--r--scripts/lib/recipetool/create_npm.py344
-rw-r--r--scripts/lib/recipetool/newappend.py89
-rw-r--r--scripts/lib/recipetool/setvar.py75
-rw-r--r--scripts/lib/scriptutils.py74
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/base.py466
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/commands/__init__.py20
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/commands/bootloader.py216
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/commands/partition.py314
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/constants.py57
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/errors.py103
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/handlers/__init__.py0
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/handlers/control.py46
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/handlers/f16.py24
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/ko.py37
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/options.py223
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/parser.py619
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/sections.py244
-rw-r--r--scripts/lib/wic/3rdparty/pykickstart/version.py168
-rw-r--r--scripts/lib/wic/__init__.py22
-rw-r--r--scripts/lib/wic/__version__.py1
-rw-r--r--scripts/lib/wic/canned-wks/common.wks.inc3
-rw-r--r--scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg27
-rw-r--r--scripts/lib/wic/canned-wks/directdisk-bootloader-config.wks8
-rw-r--r--scripts/lib/wic/canned-wks/directdisk-gpt.wks (renamed from scripts/lib/image/canned-wks/directdisk.wks)4
-rw-r--r--scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks23
-rw-r--r--scripts/lib/wic/canned-wks/directdisk.wks8
-rw-r--r--scripts/lib/wic/canned-wks/mkefidisk.wks (renamed from scripts/lib/image/canned-wks/mkefidisk.wks)4
-rw-r--r--scripts/lib/wic/canned-wks/mkhybridiso.wks7
-rw-r--r--scripts/lib/wic/canned-wks/qemux86-directdisk.wks8
-rw-r--r--scripts/lib/wic/canned-wks/sdimage-bootpart.wks (renamed from scripts/lib/image/canned-wks/sdimage-bootpart.wks)0
-rw-r--r--scripts/lib/wic/canned-wks/systemd-bootdisk.wks11
-rw-r--r--scripts/lib/wic/conf.py102
-rw-r--r--scripts/lib/wic/creator.py187
-rw-r--r--scripts/lib/wic/engine.py257
-rw-r--r--scripts/lib/wic/filemap.py564
-rw-r--r--scripts/lib/wic/help.py (renamed from scripts/lib/image/help.py)279
-rw-r--r--scripts/lib/wic/imager/__init__.py0
-rw-r--r--scripts/lib/wic/imager/baseimager.py193
-rw-r--r--scripts/lib/wic/imager/direct.py379
-rw-r--r--scripts/lib/wic/kickstart/__init__.py122
-rw-r--r--scripts/lib/wic/kickstart/custom_commands/__init__.py10
-rw-r--r--scripts/lib/wic/kickstart/custom_commands/micboot.py49
-rw-r--r--scripts/lib/wic/kickstart/custom_commands/micpartition.py57
-rw-r--r--scripts/lib/wic/kickstart/custom_commands/partition.py576
-rw-r--r--scripts/lib/wic/kickstart/custom_commands/wicboot.py57
-rw-r--r--scripts/lib/wic/ksparser.py227
-rw-r--r--scripts/lib/wic/msger.py309
-rw-r--r--scripts/lib/wic/partition.py411
-rw-r--r--scripts/lib/wic/plugin.py156
-rw-r--r--scripts/lib/wic/pluginbase.py101
-rw-r--r--scripts/lib/wic/plugins/imager/direct.py561
-rw-r--r--scripts/lib/wic/plugins/imager/direct_plugin.py98
-rw-r--r--scripts/lib/wic/plugins/source/bootimg-efi.py209
-rw-r--r--scripts/lib/wic/plugins/source/bootimg-partition.py69
-rw-r--r--scripts/lib/wic/plugins/source/bootimg-pcbios.py196
-rw-r--r--scripts/lib/wic/plugins/source/fsimage.py70
-rw-r--r--scripts/lib/wic/plugins/source/isoimage-isohybrid.py494
-rw-r--r--scripts/lib/wic/plugins/source/rawcopy.py65
-rw-r--r--scripts/lib/wic/plugins/source/rootfs.py101
-rw-r--r--scripts/lib/wic/test1
-rw-r--r--scripts/lib/wic/utils/cmdln.py1586
-rw-r--r--scripts/lib/wic/utils/errors.py47
-rw-r--r--scripts/lib/wic/utils/fs_related.py111
-rw-r--r--scripts/lib/wic/utils/misc.py253
-rw-r--r--scripts/lib/wic/utils/oe/__init__.py22
-rw-r--r--scripts/lib/wic/utils/oe/misc.py203
-rw-r--r--scripts/lib/wic/utils/partitionedfs.py375
-rw-r--r--scripts/lib/wic/utils/runner.py85
104 files changed, 11535 insertions, 9065 deletions
diff --git a/scripts/lib/argparse_oe.py b/scripts/lib/argparse_oe.py
new file mode 100644
index 0000000000..bf6eb17197
--- /dev/null
+++ b/scripts/lib/argparse_oe.py
@@ -0,0 +1,169 @@
+import sys
+import argparse
+from collections import defaultdict, OrderedDict
+
+class ArgumentUsageError(Exception):
+ """Exception class you can raise (and catch) in order to show the help"""
+ def __init__(self, message, subcommand=None):
+ self.message = message
+ self.subcommand = subcommand
+
+class ArgumentParser(argparse.ArgumentParser):
+ """Our own version of argparse's ArgumentParser"""
+ def __init__(self, *args, **kwargs):
+ kwargs.setdefault('formatter_class', OeHelpFormatter)
+ self._subparser_groups = OrderedDict()
+ super(ArgumentParser, self).__init__(*args, **kwargs)
+ self._positionals.title = 'arguments'
+ self._optionals.title = 'options'
+
+ def error(self, message):
+ """error(message: string)
+
+ Prints a help message incorporating the message to stderr and
+ exits.
+ """
+ self._print_message('%s: error: %s\n' % (self.prog, message), sys.stderr)
+ self.print_help(sys.stderr)
+ sys.exit(2)
+
+ def error_subcommand(self, message, subcommand):
+ if subcommand:
+ action = self._get_subparser_action()
+ try:
+ subparser = action._name_parser_map[subcommand]
+ except KeyError:
+ self.error('no subparser for name "%s"' % subcommand)
+ else:
+ subparser.error(message)
+
+ self.error(message)
+
+ def add_subparsers(self, *args, **kwargs):
+ if 'dest' not in kwargs:
+ kwargs['dest'] = '_subparser_name'
+
+ ret = super(ArgumentParser, self).add_subparsers(*args, **kwargs)
+ # Need a way of accessing the parent parser
+ ret._parent_parser = self
+ # Ensure our class gets instantiated
+ ret._parser_class = ArgumentSubParser
+ # Hacky way of adding a method to the subparsers object
+ ret.add_subparser_group = self.add_subparser_group
+ return ret
+
+ def add_subparser_group(self, groupname, groupdesc, order=0):
+ self._subparser_groups[groupname] = (groupdesc, order)
+
+ def parse_args(self, args=None, namespace=None):
+ """Parse arguments, using the correct subparser to show the error."""
+ args, argv = self.parse_known_args(args, namespace)
+ if argv:
+ message = 'unrecognized arguments: %s' % ' '.join(argv)
+ if self._subparsers:
+ subparser = self._get_subparser(args)
+ subparser.error(message)
+ else:
+ self.error(message)
+ sys.exit(2)
+ return args
+
+ def _get_subparser(self, args):
+ action = self._get_subparser_action()
+ if action.dest == argparse.SUPPRESS:
+ self.error('cannot get subparser, the subparser action dest is suppressed')
+
+ name = getattr(args, action.dest)
+ try:
+ return action._name_parser_map[name]
+ except KeyError:
+ self.error('no subparser for name "%s"' % name)
+
+ def _get_subparser_action(self):
+ if not self._subparsers:
+ self.error('cannot return the subparser action, no subparsers added')
+
+ for action in self._subparsers._group_actions:
+ if isinstance(action, argparse._SubParsersAction):
+ return action
+
+
+class ArgumentSubParser(ArgumentParser):
+ def __init__(self, *args, **kwargs):
+ if 'group' in kwargs:
+ self._group = kwargs.pop('group')
+ if 'order' in kwargs:
+ self._order = kwargs.pop('order')
+ super(ArgumentSubParser, self).__init__(*args, **kwargs)
+
+ def parse_known_args(self, args=None, namespace=None):
+ # This works around argparse not handling optional positional arguments being
+ # intermixed with other options. A pretty horrible hack, but we're not left
+ # with much choice given that the bug in argparse exists and it's difficult
+ # to subclass.
+ # Borrowed from http://stackoverflow.com/questions/20165843/argparse-how-to-handle-variable-number-of-arguments-nargs
+ # with an extra workaround (in format_help() below) for the positional
+ # arguments disappearing from the --help output, as well as structural tweaks.
+ # Originally simplified from http://bugs.python.org/file30204/test_intermixed.py
+ positionals = self._get_positional_actions()
+ for action in positionals:
+ # deactivate positionals
+ action.save_nargs = action.nargs
+ action.nargs = 0
+
+ namespace, remaining_args = super(ArgumentSubParser, self).parse_known_args(args, namespace)
+ for action in positionals:
+ # remove the empty positional values from namespace
+ if hasattr(namespace, action.dest):
+ delattr(namespace, action.dest)
+ for action in positionals:
+ action.nargs = action.save_nargs
+ # parse positionals
+ namespace, extras = super(ArgumentSubParser, self).parse_known_args(remaining_args, namespace)
+ return namespace, extras
+
+ def format_help(self):
+ # Quick, restore the positionals!
+ positionals = self._get_positional_actions()
+ for action in positionals:
+ if hasattr(action, 'save_nargs'):
+ action.nargs = action.save_nargs
+ return super(ArgumentParser, self).format_help()
+
+
+class OeHelpFormatter(argparse.HelpFormatter):
+ def _format_action(self, action):
+ if hasattr(action, '_get_subactions'):
+ # subcommands list
+ groupmap = defaultdict(list)
+ ordermap = {}
+ subparser_groups = action._parent_parser._subparser_groups
+ groups = sorted(subparser_groups.keys(), key=lambda item: subparser_groups[item][1], reverse=True)
+ for subaction in self._iter_indented_subactions(action):
+ parser = action._name_parser_map[subaction.dest]
+ group = getattr(parser, '_group', None)
+ groupmap[group].append(subaction)
+ if group not in groups:
+ groups.append(group)
+ order = getattr(parser, '_order', 0)
+ ordermap[subaction.dest] = order
+
+ lines = []
+ if len(groupmap) > 1:
+ groupindent = ' '
+ else:
+ groupindent = ''
+ for group in groups:
+ subactions = groupmap[group]
+ if not subactions:
+ continue
+ if groupindent:
+ if not group:
+ group = 'other'
+ groupdesc = subparser_groups.get(group, (group, 0))[0]
+ lines.append(' %s:' % groupdesc)
+ for subaction in sorted(subactions, key=lambda item: ordermap[item.dest], reverse=True):
+ lines.append('%s%s' % (groupindent, self._format_action(subaction).rstrip()))
+ return '\n'.join(lines)
+ else:
+ return super(OeHelpFormatter, self)._format_action(action)
diff --git a/scripts/lib/build_perf/__init__.py b/scripts/lib/build_perf/__init__.py
new file mode 100644
index 0000000000..1f8b729078
--- /dev/null
+++ b/scripts/lib/build_perf/__init__.py
@@ -0,0 +1,31 @@
+#
+# Copyright (c) 2017, Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+"""Build performance test library functions"""
+
+def print_table(rows, row_fmt=None):
+ """Print data table"""
+ if not rows:
+ return
+ if not row_fmt:
+ row_fmt = ['{:{wid}} '] * len(rows[0])
+
+ # Go through the data to get maximum cell widths
+ num_cols = len(row_fmt)
+ col_widths = [0] * num_cols
+ for row in rows:
+ for i, val in enumerate(row):
+ col_widths[i] = max(col_widths[i], len(str(val)))
+
+ for row in rows:
+ print(*[row_fmt[i].format(col, wid=col_widths[i]) for i, col in enumerate(row)])
+
diff --git a/scripts/lib/build_perf/html.py b/scripts/lib/build_perf/html.py
new file mode 100644
index 0000000000..578bb162ee
--- /dev/null
+++ b/scripts/lib/build_perf/html.py
@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2017, Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+"""Helper module for HTML reporting"""
+from jinja2 import Environment, PackageLoader
+
+
+env = Environment(loader=PackageLoader('build_perf', 'html'))
+
+template = env.get_template('report.html')
diff --git a/scripts/lib/build_perf/html/measurement_chart.html b/scripts/lib/build_perf/html/measurement_chart.html
new file mode 100644
index 0000000000..65f1a227ad
--- /dev/null
+++ b/scripts/lib/build_perf/html/measurement_chart.html
@@ -0,0 +1,50 @@
+<script type="text/javascript">
+ chartsDrawing += 1;
+ google.charts.setOnLoadCallback(drawChart_{{ chart_elem_id }});
+ function drawChart_{{ chart_elem_id }}() {
+ var data = new google.visualization.DataTable();
+
+ // Chart options
+ var options = {
+ theme : 'material',
+ legend: 'none',
+ hAxis: { format: '', title: 'Commit number',
+ minValue: {{ chart_opts.haxis.min }},
+ maxValue: {{ chart_opts.haxis.max }} },
+ {% if measurement.type == 'time' %}
+ vAxis: { format: 'h:mm:ss' },
+ {% else %}
+ vAxis: { format: '' },
+ {% endif %}
+ pointSize: 5,
+ chartArea: { left: 80, right: 15 },
+ };
+
+ // Define data columns
+ data.addColumn('number', 'Commit');
+ data.addColumn('{{ measurement.value_type.gv_data_type }}',
+ '{{ measurement.value_type.quantity }}');
+ // Add data rows
+ data.addRows([
+ {% for sample in measurement.samples %}
+ [{{ sample.commit_num }}, {{ sample.mean.gv_value() }}],
+ {% endfor %}
+ ]);
+
+ // Finally, draw the chart
+ chart_div = document.getElementById('{{ chart_elem_id }}');
+ var chart = new google.visualization.LineChart(chart_div);
+ google.visualization.events.addListener(chart, 'ready', function () {
+ //chart_div = document.getElementById('{{ chart_elem_id }}');
+ //chart_div.innerHTML = '<img src="' + chart.getImageURI() + '">';
+ png_div = document.getElementById('{{ chart_elem_id }}_png');
+ png_div.outerHTML = '<a id="{{ chart_elem_id }}_png" href="' + chart.getImageURI() + '">PNG</a>';
+ console.log("CHART READY: {{ chart_elem_id }}");
+ chartsDrawing -= 1;
+ if (chartsDrawing == 0)
+ console.log("ALL CHARTS READY");
+ });
+ chart.draw(data, options);
+}
+</script>
+
diff --git a/scripts/lib/build_perf/html/report.html b/scripts/lib/build_perf/html/report.html
new file mode 100644
index 0000000000..165cbb811c
--- /dev/null
+++ b/scripts/lib/build_perf/html/report.html
@@ -0,0 +1,206 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+{# Scripts, for visualization#}
+<!--START-OF-SCRIPTS-->
+<script type="text/javascript" src="https://www.gstatic.com/charts/loader.js"></script>
+<script type="text/javascript">
+google.charts.load('current', {'packages':['corechart']});
+var chartsDrawing = 0;
+</script>
+
+{# Render measurement result charts #}
+{% for test in test_data %}
+ {% if test.status == 'SUCCESS' %}
+ {% for measurement in test.measurements %}
+ {% set chart_elem_id = test.name + '_' + measurement.name + '_chart' %}
+ {% include 'measurement_chart.html' %}
+ {% endfor %}
+ {% endif %}
+{% endfor %}
+
+<!--END-OF-SCRIPTS-->
+
+{# Styles #}
+<style>
+.meta-table {
+ font-size: 14px;
+ text-align: left;
+ border-collapse: collapse;
+}
+.meta-table tr:nth-child(even){background-color: #f2f2f2}
+meta-table th, .meta-table td {
+ padding: 4px;
+}
+.summary {
+ margin: 0;
+ font-size: 14px;
+ text-align: left;
+ border-collapse: collapse;
+}
+summary th, .meta-table td {
+ padding: 4px;
+}
+.measurement {
+ padding: 8px 0px 8px 8px;
+ border: 2px solid #f0f0f0;
+ margin-bottom: 10px;
+}
+.details {
+ margin: 0;
+ font-size: 12px;
+ text-align: left;
+ border-collapse: collapse;
+}
+.details th {
+ font-weight: normal;
+ padding-right: 8px;
+}
+.preformatted {
+ font-family: monospace;
+ white-space: pre-wrap;
+ background-color: #f0f0f0;
+ margin-left: 10px;
+}
+hr {
+ color: #f0f0f0;
+}
+h2 {
+ font-size: 20px;
+ margin-bottom: 0px;
+ color: #707070;
+}
+h3 {
+ font-size: 16px;
+ margin: 0px;
+ color: #707070;
+}
+</style>
+
+<title>{{ title }}</title>
+</head>
+
+{% macro poky_link(commit) -%}
+ <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?id={{ commit }}">{{ commit[0:11] }}</a>
+{%- endmacro %}
+
+<body><div style="width: 700px">
+ {# Test metadata #}
+ <h2>General</h2>
+ <hr>
+ <table class="meta-table" style="width: 100%">
+ <tr>
+ <th></th>
+ <th>Current commit</th>
+ <th>Comparing with</th>
+ </tr>
+ {% for key, item in metadata.items() %}
+ <tr>
+ <th>{{ item.title }}</th>
+ {%if key == 'commit' %}
+ <td>{{ poky_link(item.value) }}</td>
+ <td>{{ poky_link(item.value_old) }}</td>
+ {% else %}
+ <td>{{ item.value }}</td>
+ <td>{{ item.value_old }}</td>
+ {% endif %}
+ </tr>
+ {% endfor %}
+ </table>
+
+ {# Test result summary #}
+ <h2>Test result summary</h2>
+ <hr>
+ <table class="summary" style="width: 100%">
+ {% for test in test_data %}
+ {% if loop.index is even %}
+ {% set row_style = 'style="background-color: #f2f2f2"' %}
+ {% else %}
+ {% set row_style = 'style="background-color: #ffffff"' %}
+ {% endif %}
+ <tr {{ row_style }}><td>{{ test.name }}: {{ test.description }}</td>
+ {% if test.status == 'SUCCESS' %}
+ {% for measurement in test.measurements %}
+ {# add empty cell in place of the test name#}
+ {% if loop.index > 1 %}<td></td>{% endif %}
+ {% if measurement.absdiff > 0 %}
+ {% set result_style = "color: red" %}
+ {% elif measurement.absdiff == measurement.absdiff %}
+ {% set result_style = "color: green" %}
+ {% else %}
+ {% set result_style = "color: orange" %}
+ {%endif %}
+ <td>{{ measurement.description }}</td>
+ <td style="font-weight: bold">{{ measurement.value.mean }}</td>
+ <td style="{{ result_style }}">{{ measurement.absdiff_str }}</td>
+ <td style="{{ result_style }}">{{ measurement.reldiff }}</td>
+ </tr><tr {{ row_style }}>
+ {% endfor %}
+ {% else %}
+ <td style="font-weight: bold; color: red;">{{test.status }}</td>
+ <td></td> <td></td> <td></td> <td></td>
+ {% endif %}
+ </tr>
+ {% endfor %}
+ </table>
+
+ {# Detailed test results #}
+ {% for test in test_data %}
+ <h2>{{ test.name }}: {{ test.description }}</h2>
+ <hr>
+ {% if test.status == 'SUCCESS' %}
+ {% for measurement in test.measurements %}
+ <div class="measurement">
+ <h3>{{ measurement.description }}</h3>
+ <div style="font-weight:bold;">
+ <span style="font-size: 23px;">{{ measurement.value.mean }}</span>
+ <span style="font-size: 20px; margin-left: 12px">
+ {% if measurement.absdiff > 0 %}
+ <span style="color: red">
+ {% elif measurement.absdiff == measurement.absdiff %}
+ <span style="color: green">
+ {% else %}
+ <span style="color: orange">
+ {% endif %}
+ {{ measurement.absdiff_str }} ({{measurement.reldiff}})
+ </span></span>
+ </div>
+ <table style="width: 100%">
+ <tr>
+ <td style="width: 75%">
+ {# Linechart #}
+ <div id="{{ test.name }}_{{ measurement.name }}_chart"></div>
+ </td>
+ <td>
+ {# Measurement statistics #}
+ <table class="details">
+ <tr>
+ <th>Test runs</th><td>{{ measurement.value.sample_cnt }}</td>
+ </tr><tr>
+ <th>-/+</th><td>-{{ measurement.value.minus }} / +{{ measurement.value.plus }}</td>
+ </tr><tr>
+ <th>Min</th><td>{{ measurement.value.min }}</td>
+ </tr><tr>
+ <th>Max</th><td>{{ measurement.value.max }}</td>
+ </tr><tr>
+ <th>Stdev</th><td>{{ measurement.value.stdev }}</td>
+ </tr><tr>
+ <th><div id="{{ test.name }}_{{ measurement.name }}_chart_png"></div></th>
+ </tr>
+ </table>
+ </td>
+ </tr>
+ </table>
+ </div>
+ {% endfor %}
+ {# Unsuccessful test #}
+ {% else %}
+ <span style="font-size: 150%; font-weight: bold; color: red;">{{ test.status }}
+ {% if test.err_type %}<span style="font-size: 75%; font-weight: normal">({{ test.err_type }})</span>{% endif %}
+ </span>
+ <div class="preformatted">{{ test.message }}</div>
+ {% endif %}
+ {% endfor %}
+</div></body>
+</html>
+
diff --git a/scripts/lib/build_perf/report.py b/scripts/lib/build_perf/report.py
new file mode 100644
index 0000000000..eb00ccca2d
--- /dev/null
+++ b/scripts/lib/build_perf/report.py
@@ -0,0 +1,342 @@
+#
+# Copyright (c) 2017, Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+"""Handling of build perf test reports"""
+from collections import OrderedDict, Mapping
+from datetime import datetime, timezone
+from numbers import Number
+from statistics import mean, stdev, variance
+
+
+def isofmt_to_timestamp(string):
+ """Convert timestamp string in ISO 8601 format into unix timestamp"""
+ if '.' in string:
+ dt = datetime.strptime(string, '%Y-%m-%dT%H:%M:%S.%f')
+ else:
+ dt = datetime.strptime(string, '%Y-%m-%dT%H:%M:%S')
+ return dt.replace(tzinfo=timezone.utc).timestamp()
+
+
+def metadata_xml_to_json(elem):
+ """Convert metadata xml into JSON format"""
+ assert elem.tag == 'metadata', "Invalid metadata file format"
+
+ def _xml_to_json(elem):
+ """Convert xml element to JSON object"""
+ out = OrderedDict()
+ for child in elem.getchildren():
+ key = child.attrib.get('name', child.tag)
+ if len(child):
+ out[key] = _xml_to_json(child)
+ else:
+ out[key] = child.text
+ return out
+ return _xml_to_json(elem)
+
+
+def results_xml_to_json(elem):
+ """Convert results xml into JSON format"""
+ rusage_fields = ('ru_utime', 'ru_stime', 'ru_maxrss', 'ru_minflt',
+ 'ru_majflt', 'ru_inblock', 'ru_oublock', 'ru_nvcsw',
+ 'ru_nivcsw')
+ iostat_fields = ('rchar', 'wchar', 'syscr', 'syscw', 'read_bytes',
+ 'write_bytes', 'cancelled_write_bytes')
+
+ def _read_measurement(elem):
+ """Convert measurement to JSON"""
+ data = OrderedDict()
+ data['type'] = elem.tag
+ data['name'] = elem.attrib['name']
+ data['legend'] = elem.attrib['legend']
+ values = OrderedDict()
+
+ # SYSRES measurement
+ if elem.tag == 'sysres':
+ for subel in elem:
+ if subel.tag == 'time':
+ values['start_time'] = isofmt_to_timestamp(subel.attrib['timestamp'])
+ values['elapsed_time'] = float(subel.text)
+ elif subel.tag == 'rusage':
+ rusage = OrderedDict()
+ for field in rusage_fields:
+ if 'time' in field:
+ rusage[field] = float(subel.attrib[field])
+ else:
+ rusage[field] = int(subel.attrib[field])
+ values['rusage'] = rusage
+ elif subel.tag == 'iostat':
+ values['iostat'] = OrderedDict([(f, int(subel.attrib[f]))
+ for f in iostat_fields])
+ elif subel.tag == 'buildstats_file':
+ values['buildstats_file'] = subel.text
+ else:
+ raise TypeError("Unknown sysres value element '{}'".format(subel.tag))
+ # DISKUSAGE measurement
+ elif elem.tag == 'diskusage':
+ values['size'] = int(elem.find('size').text)
+ else:
+ raise Exception("Unknown measurement tag '{}'".format(elem.tag))
+ data['values'] = values
+ return data
+
+ def _read_testcase(elem):
+ """Convert testcase into JSON"""
+ assert elem.tag == 'testcase', "Expecting 'testcase' element instead of {}".format(elem.tag)
+
+ data = OrderedDict()
+ data['name'] = elem.attrib['name']
+ data['description'] = elem.attrib['description']
+ data['status'] = 'SUCCESS'
+ data['start_time'] = isofmt_to_timestamp(elem.attrib['timestamp'])
+ data['elapsed_time'] = float(elem.attrib['time'])
+ measurements = OrderedDict()
+
+ for subel in elem.getchildren():
+ if subel.tag == 'error' or subel.tag == 'failure':
+ data['status'] = subel.tag.upper()
+ data['message'] = subel.attrib['message']
+ data['err_type'] = subel.attrib['type']
+ data['err_output'] = subel.text
+ elif subel.tag == 'skipped':
+ data['status'] = 'SKIPPED'
+ data['message'] = subel.text
+ else:
+ measurements[subel.attrib['name']] = _read_measurement(subel)
+ data['measurements'] = measurements
+ return data
+
+ def _read_testsuite(elem):
+ """Convert suite to JSON"""
+ assert elem.tag == 'testsuite', \
+ "Expecting 'testsuite' element instead of {}".format(elem.tag)
+
+ data = OrderedDict()
+ if 'hostname' in elem.attrib:
+ data['tester_host'] = elem.attrib['hostname']
+ data['start_time'] = isofmt_to_timestamp(elem.attrib['timestamp'])
+ data['elapsed_time'] = float(elem.attrib['time'])
+ tests = OrderedDict()
+
+ for case in elem.getchildren():
+ tests[case.attrib['name']] = _read_testcase(case)
+ data['tests'] = tests
+ return data
+
+ # Main function
+ assert elem.tag == 'testsuites', "Invalid test report format"
+ assert len(elem) == 1, "Too many testsuites"
+
+ return _read_testsuite(elem.getchildren()[0])
+
+
+def aggregate_metadata(metadata):
+ """Aggregate metadata into one, basically a sanity check"""
+ mutable_keys = ('pretty_name', 'version_id')
+
+ def aggregate_obj(aggregate, obj, assert_str=True):
+ """Aggregate objects together"""
+ assert type(aggregate) is type(obj), \
+ "Type mismatch: {} != {}".format(type(aggregate), type(obj))
+ if isinstance(obj, Mapping):
+ assert set(aggregate.keys()) == set(obj.keys())
+ for key, val in obj.items():
+ aggregate_obj(aggregate[key], val, key not in mutable_keys)
+ elif isinstance(obj, list):
+ assert len(aggregate) == len(obj)
+ for i, val in enumerate(obj):
+ aggregate_obj(aggregate[i], val)
+ elif not isinstance(obj, str) or (isinstance(obj, str) and assert_str):
+ assert aggregate == obj, "Data mismatch {} != {}".format(aggregate, obj)
+
+ if not metadata:
+ return {}
+
+ # Do the aggregation
+ aggregate = metadata[0].copy()
+ for testrun in metadata[1:]:
+ aggregate_obj(aggregate, testrun)
+ aggregate['testrun_count'] = len(metadata)
+ return aggregate
+
+
+def aggregate_data(data):
+ """Aggregate multiple test results JSON structures into one"""
+
+ mutable_keys = ('status', 'message', 'err_type', 'err_output')
+
+ class SampleList(list):
+ """Container for numerical samples"""
+ pass
+
+ def new_aggregate_obj(obj):
+ """Create new object for aggregate"""
+ if isinstance(obj, Number):
+ new_obj = SampleList()
+ new_obj.append(obj)
+ elif isinstance(obj, str):
+ new_obj = obj
+ else:
+ # Lists and and dicts are kept as is
+ new_obj = obj.__class__()
+ aggregate_obj(new_obj, obj)
+ return new_obj
+
+ def aggregate_obj(aggregate, obj, assert_str=True):
+ """Recursive "aggregation" of JSON objects"""
+ if isinstance(obj, Number):
+ assert isinstance(aggregate, SampleList)
+ aggregate.append(obj)
+ return
+
+ assert type(aggregate) == type(obj), \
+ "Type mismatch: {} != {}".format(type(aggregate), type(obj))
+ if isinstance(obj, Mapping):
+ for key, val in obj.items():
+ if not key in aggregate:
+ aggregate[key] = new_aggregate_obj(val)
+ else:
+ aggregate_obj(aggregate[key], val, key not in mutable_keys)
+ elif isinstance(obj, list):
+ for i, val in enumerate(obj):
+ if i >= len(aggregate):
+ aggregate[key] = new_aggregate_obj(val)
+ else:
+ aggregate_obj(aggregate[i], val)
+ elif isinstance(obj, str):
+ # Sanity check for data
+ if assert_str:
+ assert aggregate == obj, "Data mismatch {} != {}".format(aggregate, obj)
+ else:
+ raise Exception("BUG: unable to aggregate '{}' ({})".format(type(obj), str(obj)))
+
+ if not data:
+ return {}
+
+ # Do the aggregation
+ aggregate = data[0].__class__()
+ for testrun in data:
+ aggregate_obj(aggregate, testrun)
+ return aggregate
+
+
+class MeasurementVal(float):
+ """Base class representing measurement values"""
+ gv_data_type = 'number'
+
+ def gv_value(self):
+ """Value formatting for visualization"""
+ if self != self:
+ return "null"
+ else:
+ return self
+
+
+class TimeVal(MeasurementVal):
+ """Class representing time values"""
+ quantity = 'time'
+ gv_title = 'elapsed time'
+ gv_data_type = 'timeofday'
+
+ def hms(self):
+ """Split time into hours, minutes and seconeds"""
+ hhh = int(abs(self) / 3600)
+ mmm = int((abs(self) % 3600) / 60)
+ sss = abs(self) % 60
+ return hhh, mmm, sss
+
+ def __str__(self):
+ if self != self:
+ return "nan"
+ hh, mm, ss = self.hms()
+ sign = '-' if self < 0 else ''
+ if hh > 0:
+ return '{}{:d}:{:02d}:{:02.0f}'.format(sign, hh, mm, ss)
+ elif mm > 0:
+ return '{}{:d}:{:04.1f}'.format(sign, mm, ss)
+ elif ss > 1:
+ return '{}{:.1f} s'.format(sign, ss)
+ else:
+ return '{}{:.2f} s'.format(sign, ss)
+
+ def gv_value(self):
+ """Value formatting for visualization"""
+ if self != self:
+ return "null"
+ hh, mm, ss = self.hms()
+ return [hh, mm, int(ss), int(ss*1000) % 1000]
+
+
+class SizeVal(MeasurementVal):
+ """Class representing time values"""
+ quantity = 'size'
+ gv_title = 'size in MiB'
+ gv_data_type = 'number'
+
+ def __str__(self):
+ if self != self:
+ return "nan"
+ if abs(self) < 1024:
+ return '{:.1f} kiB'.format(self)
+ elif abs(self) < 1048576:
+ return '{:.2f} MiB'.format(self / 1024)
+ else:
+ return '{:.2f} GiB'.format(self / 1048576)
+
+ def gv_value(self):
+ """Value formatting for visualization"""
+ if self != self:
+ return "null"
+ return self / 1024
+
+def measurement_stats(meas, prefix=''):
+ """Get statistics of a measurement"""
+ if not meas:
+ return {prefix + 'sample_cnt': 0,
+ prefix + 'mean': MeasurementVal('nan'),
+ prefix + 'stdev': MeasurementVal('nan'),
+ prefix + 'variance': MeasurementVal('nan'),
+ prefix + 'min': MeasurementVal('nan'),
+ prefix + 'max': MeasurementVal('nan'),
+ prefix + 'minus': MeasurementVal('nan'),
+ prefix + 'plus': MeasurementVal('nan')}
+
+ stats = {'name': meas['name']}
+ if meas['type'] == 'sysres':
+ val_cls = TimeVal
+ values = meas['values']['elapsed_time']
+ elif meas['type'] == 'diskusage':
+ val_cls = SizeVal
+ values = meas['values']['size']
+ else:
+ raise Exception("Unknown measurement type '{}'".format(meas['type']))
+ stats['val_cls'] = val_cls
+ stats['quantity'] = val_cls.quantity
+ stats[prefix + 'sample_cnt'] = len(values)
+
+ mean_val = val_cls(mean(values))
+ min_val = val_cls(min(values))
+ max_val = val_cls(max(values))
+
+ stats[prefix + 'mean'] = mean_val
+ if len(values) > 1:
+ stats[prefix + 'stdev'] = val_cls(stdev(values))
+ stats[prefix + 'variance'] = val_cls(variance(values))
+ else:
+ stats[prefix + 'stdev'] = float('nan')
+ stats[prefix + 'variance'] = float('nan')
+ stats[prefix + 'min'] = min_val
+ stats[prefix + 'max'] = max_val
+ stats[prefix + 'minus'] = val_cls(mean_val - min_val)
+ stats[prefix + 'plus'] = val_cls(max_val - mean_val)
+
+ return stats
+
diff --git a/scripts/lib/build_perf/scrape-html-report.js b/scripts/lib/build_perf/scrape-html-report.js
new file mode 100644
index 0000000000..05a1f57001
--- /dev/null
+++ b/scripts/lib/build_perf/scrape-html-report.js
@@ -0,0 +1,56 @@
+var fs = require('fs');
+var system = require('system');
+var page = require('webpage').create();
+
+// Examine console log for message from chart drawing
+page.onConsoleMessage = function(msg) {
+ console.log(msg);
+ if (msg === "ALL CHARTS READY") {
+ window.charts_ready = true;
+ }
+ else if (msg.slice(0, 11) === "CHART READY") {
+ var chart_id = msg.split(" ")[2];
+ console.log('grabbing ' + chart_id);
+ var png_data = page.evaluate(function (chart_id) {
+ var chart_div = document.getElementById(chart_id + '_png');
+ return chart_div.outerHTML;
+ }, chart_id);
+ fs.write(args[2] + '/' + chart_id + '.png', png_data, 'w');
+ }
+};
+
+// Check command line arguments
+var args = system.args;
+if (args.length != 3) {
+ console.log("USAGE: " + args[0] + " REPORT_HTML OUT_DIR\n");
+ phantom.exit(1);
+}
+
+// Open the web page
+page.open(args[1], function(status) {
+ if (status == 'fail') {
+ console.log("Failed to open file '" + args[1] + "'");
+ phantom.exit(1);
+ }
+});
+
+// Check status every 100 ms
+interval = window.setInterval(function () {
+ //console.log('waiting');
+ if (window.charts_ready) {
+ clearTimeout(timer);
+ clearInterval(interval);
+
+ var fname = args[1].replace(/\/+$/, "").split("/").pop()
+ console.log("saving " + fname);
+ fs.write(args[2] + '/' + fname, page.content, 'w');
+ phantom.exit(0);
+ }
+}, 100);
+
+// Time-out after 10 seconds
+timer = window.setTimeout(function () {
+ clearInterval(interval);
+ console.log("ERROR: timeout");
+ phantom.exit(1);
+}, 10000);
diff --git a/scripts/lib/compatlayer/__init__.py b/scripts/lib/compatlayer/__init__.py
new file mode 100644
index 0000000000..e35f8c0d32
--- /dev/null
+++ b/scripts/lib/compatlayer/__init__.py
@@ -0,0 +1,308 @@
+# Yocto Project compatibility layer tool
+#
+# Copyright (C) 2017 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import subprocess
+from enum import Enum
+
+import bb.tinfoil
+
+class LayerType(Enum):
+ BSP = 0
+ DISTRO = 1
+ SOFTWARE = 2
+ ERROR_NO_LAYER_CONF = 98
+ ERROR_BSP_DISTRO = 99
+
+def _get_configurations(path):
+ configs = []
+
+ for f in os.listdir(path):
+ file_path = os.path.join(path, f)
+ if os.path.isfile(file_path) and f.endswith('.conf'):
+ configs.append(f[:-5]) # strip .conf
+ return configs
+
+def _get_layer_collections(layer_path, lconf=None, data=None):
+ import bb.parse
+ import bb.data
+
+ if lconf is None:
+ lconf = os.path.join(layer_path, 'conf', 'layer.conf')
+
+ if data is None:
+ ldata = bb.data.init()
+ bb.parse.init_parser(ldata)
+ else:
+ ldata = data.createCopy()
+
+ ldata.setVar('LAYERDIR', layer_path)
+ try:
+ ldata = bb.parse.handle(lconf, ldata, include=True)
+ except BaseException as exc:
+ raise LayerError(exc)
+ ldata.expandVarref('LAYERDIR')
+
+ collections = (ldata.getVar('BBFILE_COLLECTIONS', True) or '').split()
+ if not collections:
+ name = os.path.basename(layer_path)
+ collections = [name]
+
+ collections = {c: {} for c in collections}
+ for name in collections:
+ priority = ldata.getVar('BBFILE_PRIORITY_%s' % name, True)
+ pattern = ldata.getVar('BBFILE_PATTERN_%s' % name, True)
+ depends = ldata.getVar('LAYERDEPENDS_%s' % name, True)
+ collections[name]['priority'] = priority
+ collections[name]['pattern'] = pattern
+ collections[name]['depends'] = depends
+
+ return collections
+
+def _detect_layer(layer_path):
+ """
+ Scans layer directory to detect what type of layer
+ is BSP, Distro or Software.
+
+ Returns a dictionary with layer name, type and path.
+ """
+
+ layer = {}
+ layer_name = os.path.basename(layer_path)
+
+ layer['name'] = layer_name
+ layer['path'] = layer_path
+ layer['conf'] = {}
+
+ if not os.path.isfile(os.path.join(layer_path, 'conf', 'layer.conf')):
+ layer['type'] = LayerType.ERROR_NO_LAYER_CONF
+ return layer
+
+ machine_conf = os.path.join(layer_path, 'conf', 'machine')
+ distro_conf = os.path.join(layer_path, 'conf', 'distro')
+
+ is_bsp = False
+ is_distro = False
+
+ if os.path.isdir(machine_conf):
+ machines = _get_configurations(machine_conf)
+ if machines:
+ is_bsp = True
+
+ if os.path.isdir(distro_conf):
+ distros = _get_configurations(distro_conf)
+ if distros:
+ is_distro = True
+
+ if is_bsp and is_distro:
+ layer['type'] = LayerType.ERROR_BSP_DISTRO
+ elif is_bsp:
+ layer['type'] = LayerType.BSP
+ layer['conf']['machines'] = machines
+ elif is_distro:
+ layer['type'] = LayerType.DISTRO
+ layer['conf']['distros'] = distros
+ else:
+ layer['type'] = LayerType.SOFTWARE
+
+ layer['collections'] = _get_layer_collections(layer['path'])
+
+ return layer
+
+def detect_layers(layer_directories, no_auto):
+ layers = []
+
+ for directory in layer_directories:
+ directory = os.path.realpath(directory)
+ if directory[-1] == '/':
+ directory = directory[0:-1]
+
+ if no_auto:
+ conf_dir = os.path.join(directory, 'conf')
+ if os.path.isdir(conf_dir):
+ layer = _detect_layer(directory)
+ if layer:
+ layers.append(layer)
+ else:
+ for root, dirs, files in os.walk(directory):
+ dir_name = os.path.basename(root)
+ conf_dir = os.path.join(root, 'conf')
+ if os.path.isdir(conf_dir):
+ layer = _detect_layer(root)
+ if layer:
+ layers.append(layer)
+
+ return layers
+
+def _find_layer_depends(depend, layers):
+ for layer in layers:
+ for collection in layer['collections']:
+ if depend == collection:
+ return layer
+ return None
+
+def add_layer_dependencies(bblayersconf, layer, layers, logger):
+ def recurse_dependencies(depends, layer, layers, logger, ret = []):
+ logger.debug('Processing dependencies %s for layer %s.' % \
+ (depends, layer['name']))
+
+ for depend in depends.split():
+ # core (oe-core) is suppose to be provided
+ if depend == 'core':
+ continue
+
+ layer_depend = _find_layer_depends(depend, layers)
+ if not layer_depend:
+ logger.error('Layer %s depends on %s and isn\'t found.' % \
+ (layer['name'], depend))
+ ret = None
+ continue
+
+ # We keep processing, even if ret is None, this allows us to report
+ # multiple errors at once
+ if ret is not None and layer_depend not in ret:
+ ret.append(layer_depend)
+
+ # Recursively process...
+ if 'collections' not in layer_depend:
+ continue
+
+ for collection in layer_depend['collections']:
+ collect_deps = layer_depend['collections'][collection]['depends']
+ if not collect_deps:
+ continue
+ ret = recurse_dependencies(collect_deps, layer_depend, layers, logger, ret)
+
+ return ret
+
+ layer_depends = []
+ for collection in layer['collections']:
+ depends = layer['collections'][collection]['depends']
+ if not depends:
+ continue
+
+ layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends)
+
+ # Note: [] (empty) is allowed, None is not!
+ if layer_depends is None:
+ return False
+ else:
+ for layer_depend in layer_depends:
+ logger.info('Adding layer dependency %s' % layer_depend['name'])
+ with open(bblayersconf, 'a+') as f:
+ f.write("\nBBLAYERS += \"%s\"\n" % layer_depend['path'])
+ return True
+
+def add_layer(bblayersconf, layer, layers, logger):
+ logger.info('Adding layer %s' % layer['name'])
+ with open(bblayersconf, 'a+') as f:
+ f.write("\nBBLAYERS += \"%s\"\n" % layer['path'])
+
+ return True
+
+def check_command(error_msg, cmd):
+ '''
+ Run a command under a shell, capture stdout and stderr in a single stream,
+ throw an error when command returns non-zero exit code. Returns the output.
+ '''
+
+ p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ output, _ = p.communicate()
+ if p.returncode:
+ msg = "%s\nCommand: %s\nOutput:\n%s" % (error_msg, cmd, output.decode('utf-8'))
+ raise RuntimeError(msg)
+ return output
+
+def get_signatures(builddir, failsafe=False, machine=None):
+ import re
+
+ # some recipes needs to be excluded like meta-world-pkgdata
+ # because a layer can add recipes to a world build so signature
+ # will be change
+ exclude_recipes = ('meta-world-pkgdata',)
+
+ sigs = {}
+ tune2tasks = {}
+
+ cmd = ''
+ if machine:
+ cmd += 'MACHINE=%s ' % machine
+ cmd += 'bitbake '
+ if failsafe:
+ cmd += '-k '
+ cmd += '-S none world'
+ sigs_file = os.path.join(builddir, 'locked-sigs.inc')
+ if os.path.exists(sigs_file):
+ os.unlink(sigs_file)
+ try:
+ check_command('Generating signatures failed. This might be due to some parse error and/or general layer incompatibilities.',
+ cmd)
+ except RuntimeError as ex:
+ if failsafe and os.path.exists(sigs_file):
+ # Ignore the error here. Most likely some recipes active
+ # in a world build lack some dependencies. There is a
+ # separate test_machine_world_build which exposes the
+ # failure.
+ pass
+ else:
+ raise
+
+ sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$")
+ tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*")
+ current_tune = None
+ with open(sigs_file, 'r') as f:
+ for line in f.readlines():
+ line = line.strip()
+ t = tune_regex.search(line)
+ if t:
+ current_tune = t.group('tune')
+ s = sig_regex.match(line)
+ if s:
+ exclude = False
+ for er in exclude_recipes:
+ (recipe, task) = s.group('task').split(':')
+ if er == recipe:
+ exclude = True
+ break
+ if exclude:
+ continue
+
+ sigs[s.group('task')] = s.group('hash')
+ tune2tasks.setdefault(current_tune, []).append(s.group('task'))
+
+ if not sigs:
+ raise RuntimeError('Can\'t load signatures from %s' % sigs_file)
+
+ return (sigs, tune2tasks)
+
+def get_depgraph(targets=['world']):
+ '''
+ Returns the dependency graph for the given target(s).
+ The dependency graph is taken directly from DepTreeEvent.
+ '''
+ depgraph = None
+ with bb.tinfoil.Tinfoil() as tinfoil:
+ tinfoil.prepare(config_only=False)
+ tinfoil.set_event_mask(['bb.event.NoProvider', 'bb.event.DepTreeGenerated', 'bb.command.CommandCompleted'])
+ if not tinfoil.run_command('generateDepTreeEvent', targets, 'do_build'):
+ raise RuntimeError('starting generateDepTreeEvent failed')
+ while True:
+ event = tinfoil.wait_event(timeout=1000)
+ if event:
+ if isinstance(event, bb.command.CommandFailed):
+ raise RuntimeError('Generating dependency information failed: %s' % event.error)
+ elif isinstance(event, bb.command.CommandCompleted):
+ break
+ elif isinstance(event, bb.event.NoProvider):
+ if event._reasons:
+ raise RuntimeError('Nothing provides %s: %s' % (event._item, event._reasons))
+ else:
+ raise RuntimeError('Nothing provides %s.' % (event._item))
+ elif isinstance(event, bb.event.DepTreeGenerated):
+ depgraph = event._depgraph
+
+ if depgraph is None:
+ raise RuntimeError('Could not retrieve the depgraph.')
+ return depgraph
diff --git a/scripts/lib/compatlayer/case.py b/scripts/lib/compatlayer/case.py
new file mode 100644
index 0000000000..54ce78aa60
--- /dev/null
+++ b/scripts/lib/compatlayer/case.py
@@ -0,0 +1,7 @@
+# Copyright (C) 2017 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.core.case import OETestCase
+
+class OECompatLayerTestCase(OETestCase):
+ pass
diff --git a/scripts/lib/wic/3rdparty/pykickstart/__init__.py b/scripts/lib/compatlayer/cases/__init__.py
index e69de29bb2..e69de29bb2 100644
--- a/scripts/lib/wic/3rdparty/pykickstart/__init__.py
+++ b/scripts/lib/compatlayer/cases/__init__.py
diff --git a/scripts/lib/compatlayer/cases/bsp.py b/scripts/lib/compatlayer/cases/bsp.py
new file mode 100644
index 0000000000..43efae406f
--- /dev/null
+++ b/scripts/lib/compatlayer/cases/bsp.py
@@ -0,0 +1,204 @@
+# Copyright (C) 2017 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import unittest
+
+from compatlayer import LayerType, get_signatures, check_command, get_depgraph
+from compatlayer.case import OECompatLayerTestCase
+
+class BSPCompatLayer(OECompatLayerTestCase):
+ @classmethod
+ def setUpClass(self):
+ if self.tc.layer['type'] != LayerType.BSP:
+ raise unittest.SkipTest("BSPCompatLayer: Layer %s isn't BSP one." %\
+ self.tc.layer['name'])
+
+ def test_bsp_defines_machines(self):
+ self.assertTrue(self.tc.layer['conf']['machines'],
+ "Layer is BSP but doesn't defines machines.")
+
+ def test_bsp_no_set_machine(self):
+ from oeqa.utils.commands import get_bb_var
+
+ machine = get_bb_var('MACHINE')
+ self.assertEqual(self.td['bbvars']['MACHINE'], machine,
+ msg="Layer %s modified machine %s -> %s" % \
+ (self.tc.layer['name'], self.td['bbvars']['MACHINE'], machine))
+
+
+ def test_machine_world(self):
+ '''
+ "bitbake world" is expected to work regardless which machine is selected.
+ BSP layers sometimes break that by enabling a recipe for a certain machine
+ without checking whether that recipe actually can be built in the current
+ distro configuration (for example, OpenGL might not enabled).
+
+ This test iterates over all machines. It would be nicer to instantiate
+ it once per machine. It merely checks for errors during parse
+ time. It does not actually attempt to build anything.
+ '''
+
+ if not self.td['machines']:
+ self.skipTest('No machines set with --machines.')
+ msg = []
+ for machine in self.td['machines']:
+ # In contrast to test_machine_signatures() below, errors are fatal here.
+ try:
+ get_signatures(self.td['builddir'], failsafe=False, machine=machine)
+ except RuntimeError as ex:
+ msg.append(str(ex))
+ if msg:
+ msg.insert(0, 'The following machines broke a world build:')
+ self.fail('\n'.join(msg))
+
+ def test_machine_signatures(self):
+ '''
+ Selecting a machine may only affect the signature of tasks that are specific
+ to that machine. In other words, when MACHINE=A and MACHINE=B share a recipe
+ foo and the output of foo, then both machine configurations must build foo
+ in exactly the same way. Otherwise it is not possible to use both machines
+ in the same distribution.
+
+ This criteria can only be tested by testing different machines in combination,
+ i.e. one main layer, potentially several additional BSP layers and an explicit
+ choice of machines:
+ yocto-compat-layer --additional-layers .../meta-intel --machines intel-corei7-64 imx6slevk -- .../meta-freescale
+ '''
+
+ if not self.td['machines']:
+ self.skipTest('No machines set with --machines.')
+
+ # Collect signatures for all machines that we are testing
+ # and merge that into a hash:
+ # tune -> task -> signature -> list of machines with that combination
+ #
+ # It is an error if any tune/task pair has more than one signature,
+ # because that implies that the machines that caused those different
+ # signatures do not agree on how to execute the task.
+ tunes = {}
+ # Preserve ordering of machines as chosen by the user.
+ for machine in self.td['machines']:
+ curr_sigs, tune2tasks = get_signatures(self.td['builddir'], failsafe=True, machine=machine)
+ # Invert the tune -> [tasks] mapping.
+ tasks2tune = {}
+ for tune, tasks in tune2tasks.items():
+ for task in tasks:
+ tasks2tune[task] = tune
+ for task, sighash in curr_sigs.items():
+ tunes.setdefault(tasks2tune[task], {}).setdefault(task, {}).setdefault(sighash, []).append(machine)
+
+ msg = []
+ pruned = 0
+ last_line_key = None
+ # do_fetch, do_unpack, ..., do_build
+ taskname_list = []
+ if tunes:
+ # The output below is most useful when we start with tasks that are at
+ # the bottom of the dependency chain, i.e. those that run first. If
+ # those tasks differ, the rest also does.
+ #
+ # To get an ordering of tasks, we do a topological sort of the entire
+ # depgraph for the base configuration, then on-the-fly flatten that list by stripping
+ # out the recipe names and removing duplicates. The base configuration
+ # is not necessarily representative, but should be close enough. Tasks
+ # that were not encountered get a default priority.
+ depgraph = get_depgraph()
+ depends = depgraph['tdepends']
+ WHITE = 1
+ GRAY = 2
+ BLACK = 3
+ color = {}
+ found = set()
+ def visit(task):
+ color[task] = GRAY
+ for dep in depends.get(task, ()):
+ if color.setdefault(dep, WHITE) == WHITE:
+ visit(dep)
+ color[task] = BLACK
+ pn, taskname = task.rsplit('.', 1)
+ if taskname not in found:
+ taskname_list.append(taskname)
+ found.add(taskname)
+ for task in depends.keys():
+ if color.setdefault(task, WHITE) == WHITE:
+ visit(task)
+
+ taskname_order = dict([(task, index) for index, task in enumerate(taskname_list) ])
+ def task_key(task):
+ pn, taskname = task.rsplit(':', 1)
+ return (pn, taskname_order.get(taskname, len(taskname_list)), taskname)
+
+ for tune in sorted(tunes.keys()):
+ tasks = tunes[tune]
+ # As for test_signatures it would be nicer to sort tasks
+ # by dependencies here, but that is harder because we have
+ # to report on tasks from different machines, which might
+ # have different dependencies. We resort to pruning the
+ # output by reporting only one task per recipe if the set
+ # of machines matches.
+ #
+ # "bitbake-diffsigs -t -s" is intelligent enough to print
+ # diffs recursively, so often it does not matter that much
+ # if we don't pick the underlying difference
+ # here. However, sometimes recursion fails
+ # (https://bugzilla.yoctoproject.org/show_bug.cgi?id=6428).
+ #
+ # To mitigate that a bit, we use a hard-coded ordering of
+ # tasks that represents how they normally run and prefer
+ # to print the ones that run first.
+ for task in sorted(tasks.keys(), key=task_key):
+ signatures = tasks[task]
+ # do_build can be ignored: it is know to have
+ # different signatures in some cases, for example in
+ # the allarch ca-certificates due to RDEPENDS=openssl.
+ # That particular dependency is whitelisted via
+ # SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS, but still shows up
+ # in the sstate signature hash because filtering it
+ # out would be hard and running do_build multiple
+ # times doesn't really matter.
+ if len(signatures.keys()) > 1 and \
+ not task.endswith(':do_build'):
+ # Error!
+ #
+ # Sort signatures by machines, because the hex values don't mean anything.
+ # => all-arch adwaita-icon-theme:do_build: 1234... (beaglebone, qemux86) != abcdf... (qemux86-64)
+ #
+ # Skip the line if it is covered already by the predecessor (same pn, same sets of machines).
+ pn, taskname = task.rsplit(':', 1)
+ next_line_key = (pn, sorted(signatures.values()))
+ if next_line_key != last_line_key:
+ line = ' %s %s: ' % (tune, task)
+ line += ' != '.join(['%s (%s)' % (signature, ', '.join([m for m in signatures[signature]])) for
+ signature in sorted(signatures.keys(), key=lambda s: signatures[s])])
+ last_line_key = next_line_key
+ msg.append(line)
+ # Randomly pick two mismatched signatures and remember how to invoke
+ # bitbake-diffsigs for them.
+ iterator = iter(signatures.items())
+ a = next(iterator)
+ b = next(iterator)
+ diffsig_machines = '(%s) != (%s)' % (', '.join(a[1]), ', '.join(b[1]))
+ diffsig_params = '-t %s %s -s %s %s' % (pn, taskname, a[0], b[0])
+ else:
+ pruned += 1
+
+ if msg:
+ msg.insert(0, 'The machines have conflicting signatures for some shared tasks:')
+ if pruned > 0:
+ msg.append('')
+ msg.append('%d tasks where not listed because some other task of the recipe already differed.' % pruned)
+ msg.append('It is likely that differences from different recipes also have the same root cause.')
+ msg.append('')
+ # Explain how to investigate...
+ msg.append('To investigate, run bitbake-diffsigs -t recipename taskname -s fromsig tosig.')
+ cmd = 'bitbake-diffsigs %s' % diffsig_params
+ msg.append('Example: %s in the last line' % diffsig_machines)
+ msg.append('Command: %s' % cmd)
+ # ... and actually do it automatically for that example, but without aborting
+ # when that fails.
+ try:
+ output = check_command('Comparing signatures failed.', cmd).decode('utf-8')
+ except RuntimeError as ex:
+ output = str(ex)
+ msg.extend([' ' + line for line in output.splitlines()])
+ self.fail('\n'.join(msg))
diff --git a/scripts/lib/compatlayer/cases/common.py b/scripts/lib/compatlayer/cases/common.py
new file mode 100644
index 0000000000..8eeada9b1e
--- /dev/null
+++ b/scripts/lib/compatlayer/cases/common.py
@@ -0,0 +1,91 @@
+# Copyright (C) 2017 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import unittest
+from compatlayer import get_signatures, LayerType, check_command, get_depgraph
+from compatlayer.case import OECompatLayerTestCase
+
+class CommonCompatLayer(OECompatLayerTestCase):
+ def test_readme(self):
+ readme_file = os.path.join(self.tc.layer['path'], 'README')
+ self.assertTrue(os.path.isfile(readme_file),
+ msg="Layer doesn't contains README file.")
+
+ data = ''
+ with open(readme_file, 'r') as f:
+ data = f.read()
+ self.assertTrue(data,
+ msg="Layer contains README file but is empty.")
+
+ def test_parse(self):
+ check_command('Layer %s failed to parse.' % self.tc.layer['name'],
+ 'bitbake -p')
+
+ def test_show_environment(self):
+ check_command('Layer %s failed to show environment.' % self.tc.layer['name'],
+ 'bitbake -e')
+
+ def test_signatures(self):
+ if self.tc.layer['type'] == LayerType.SOFTWARE:
+ raise unittest.SkipTest("Layer %s isn't BSP or DISTRO one." \
+ % self.tc.layer['name'])
+
+ # task -> (old signature, new signature)
+ sig_diff = {}
+ curr_sigs, _ = get_signatures(self.td['builddir'], failsafe=True)
+ for task in self.td['sigs']:
+ if task in curr_sigs and \
+ self.td['sigs'][task] != curr_sigs[task]:
+ sig_diff[task] = (self.td['sigs'][task], curr_sigs[task])
+
+ if sig_diff:
+ # Beware, depgraph uses task=<pn>.<taskname> whereas get_signatures()
+ # uses <pn>:<taskname>. Need to convert sometimes. The output follows
+ # the convention from get_signatures() because that seems closer to
+ # normal bitbake output.
+ def sig2graph(task):
+ pn, taskname = task.rsplit(':', 1)
+ return pn + '.' + taskname
+ def graph2sig(task):
+ pn, taskname = task.rsplit('.', 1)
+ return pn + ':' + taskname
+ depgraph = get_depgraph()
+ depends = depgraph['tdepends']
+
+ # If a task A has a changed signature, but none of its
+ # dependencies, then we need to report it because it is
+ # the one which introduces a change. Any task depending on
+ # A (directly or indirectly) will also have a changed
+ # signature, but we don't need to report it. It might have
+ # its own changes, which will become apparent once the
+ # issues that we do report are fixed and the test gets run
+ # again.
+ sig_diff_filtered = []
+ for task, (old_sig, new_sig) in sig_diff.items():
+ deps_tainted = False
+ for dep in depends.get(sig2graph(task), ()):
+ if graph2sig(dep) in sig_diff:
+ deps_tainted = True
+ break
+ if not deps_tainted:
+ sig_diff_filtered.append((task, old_sig, new_sig))
+
+ msg = []
+ msg.append('Layer %s changed %d signatures, initial differences (first hash without, second with layer):' %
+ (self.tc.layer['name'], len(sig_diff)))
+ for diff in sorted(sig_diff_filtered):
+ recipe, taskname = diff[0].rsplit(':', 1)
+ cmd = 'bitbake-diffsigs --task %s %s --signature %s %s' % \
+ (recipe, taskname, diff[1], diff[2])
+ msg.append(' %s: %s -> %s' % diff)
+ msg.append(' %s' % cmd)
+ try:
+ output = check_command('Determining signature difference failed.',
+ cmd).decode('utf-8')
+ except RuntimeError as error:
+ output = str(error)
+ if output:
+ msg.extend([' ' + line for line in output.splitlines()])
+ msg.append('')
+ self.fail('\n'.join(msg))
diff --git a/scripts/lib/compatlayer/cases/distro.py b/scripts/lib/compatlayer/cases/distro.py
new file mode 100644
index 0000000000..523acc1e78
--- /dev/null
+++ b/scripts/lib/compatlayer/cases/distro.py
@@ -0,0 +1,26 @@
+# Copyright (C) 2017 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import unittest
+
+from compatlayer import LayerType
+from compatlayer.case import OECompatLayerTestCase
+
+class DistroCompatLayer(OECompatLayerTestCase):
+ @classmethod
+ def setUpClass(self):
+ if self.tc.layer['type'] != LayerType.DISTRO:
+ raise unittest.SkipTest("DistroCompatLayer: Layer %s isn't Distro one." %\
+ self.tc.layer['name'])
+
+ def test_distro_defines_distros(self):
+ self.assertTrue(self.tc.layer['conf']['distros'],
+ "Layer is BSP but doesn't defines machines.")
+
+ def test_distro_no_set_distros(self):
+ from oeqa.utils.commands import get_bb_var
+
+ distro = get_bb_var('DISTRO')
+ self.assertEqual(self.td['bbvars']['DISTRO'], distro,
+ msg="Layer %s modified distro %s -> %s" % \
+ (self.tc.layer['name'], self.td['bbvars']['DISTRO'], distro))
diff --git a/scripts/lib/compatlayer/context.py b/scripts/lib/compatlayer/context.py
new file mode 100644
index 0000000000..4932238798
--- /dev/null
+++ b/scripts/lib/compatlayer/context.py
@@ -0,0 +1,14 @@
+# Copyright (C) 2017 Intel Corporation
+# Released under the MIT license (see COPYING.MIT)
+
+import os
+import sys
+import glob
+import re
+
+from oeqa.core.context import OETestContext
+
+class CompatLayerTestContext(OETestContext):
+ def __init__(self, td=None, logger=None, layer=None):
+ super(CompatLayerTestContext, self).__init__(td, logger)
+ self.layer = layer
diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py
index 4c0d5397dc..d646b0cf63 100644
--- a/scripts/lib/devtool/__init__.py
+++ b/scripts/lib/devtool/__init__.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Development tool - utility functions for plugins
#
@@ -16,16 +16,26 @@
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
+"""Devtool plugins module"""
import os
import sys
import subprocess
import logging
+import re
+import codecs
logger = logging.getLogger('devtool')
+class DevtoolError(Exception):
+ """Exception for handling devtool errors"""
+ def __init__(self, message, exitcode=1):
+ super(DevtoolError, self).__init__(message)
+ self.exitcode = exitcode
+
+
def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
+ """Run a program in bitbake build context"""
import bb
if not 'cwd' in options:
options["cwd"] = builddir
@@ -49,17 +59,19 @@ def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
return bb.process.run('%s%s' % (init_prefix, cmd), **options)
def exec_watch(cmd, **options):
+ """Run program with stdout shown on sys.stdout"""
import bb
- if isinstance(cmd, basestring) and not "shell" in options:
+ if isinstance(cmd, str) and not "shell" in options:
options["shell"] = True
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options
)
+ reader = codecs.getreader('utf-8')(process.stdout)
buf = ''
while True:
- out = process.stdout.read(1)
+ out = reader.read(1, 1)
if out:
sys.stdout.write(out)
sys.stdout.flush()
@@ -72,17 +84,211 @@ def exec_watch(cmd, **options):
return buf, None
-def setup_tinfoil():
+def exec_fakeroot(d, cmd, **kwargs):
+ """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions"""
+ # Grab the command and check it actually exists
+ fakerootcmd = d.getVar('FAKEROOTCMD')
+ if not os.path.exists(fakerootcmd):
+ logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built')
+ return 2
+ # Set up the appropriate environment
+ newenv = dict(os.environ)
+ fakerootenv = d.getVar('FAKEROOTENV')
+ for varvalue in fakerootenv.split():
+ if '=' in varvalue:
+ splitval = varvalue.split('=', 1)
+ newenv[splitval[0]] = splitval[1]
+ return subprocess.call("%s %s" % (fakerootcmd, cmd), env=newenv, **kwargs)
+
+def setup_tinfoil(config_only=False, basepath=None, tracking=False):
+ """Initialize tinfoil api from bitbake"""
import scriptpath
- bitbakepath = scriptpath.add_bitbake_lib_path()
- if not bitbakepath:
- logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
- sys.exit(1)
-
- import bb.tinfoil
- import logging
- tinfoil = bb.tinfoil.Tinfoil()
- tinfoil.prepare(False)
- tinfoil.logger.setLevel(logging.WARNING)
+ orig_cwd = os.path.abspath(os.curdir)
+ try:
+ if basepath:
+ os.chdir(basepath)
+ bitbakepath = scriptpath.add_bitbake_lib_path()
+ if not bitbakepath:
+ logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
+ sys.exit(1)
+
+ import bb.tinfoil
+ tinfoil = bb.tinfoil.Tinfoil(tracking=tracking)
+ try:
+ tinfoil.prepare(config_only)
+ tinfoil.logger.setLevel(logger.getEffectiveLevel())
+ except bb.tinfoil.TinfoilUIException:
+ tinfoil.shutdown()
+ raise DevtoolError('Failed to start bitbake environment')
+ except:
+ tinfoil.shutdown()
+ raise
+ finally:
+ os.chdir(orig_cwd)
return tinfoil
+def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
+ """Parse the specified recipe"""
+ try:
+ recipefile = tinfoil.get_recipe_file(pn)
+ except bb.providers.NoProvider as e:
+ logger.error(str(e))
+ return None
+ if appends:
+ append_files = tinfoil.get_file_appends(recipefile)
+ if filter_workspace:
+ # Filter out appends from the workspace
+ append_files = [path for path in append_files if
+ not path.startswith(config.workspace_path)]
+ else:
+ append_files = None
+ try:
+ rd = tinfoil.parse_recipe_file(recipefile, appends, append_files)
+ except Exception as e:
+ logger.error(str(e))
+ return None
+ return rd
+
+def check_workspace_recipe(workspace, pn, checksrc=True, bbclassextend=False):
+ """
+ Check that a recipe is in the workspace and (optionally) that source
+ is present.
+ """
+
+ workspacepn = pn
+
+ for recipe, value in workspace.items():
+ if recipe == pn:
+ break
+ if bbclassextend:
+ recipefile = value['recipefile']
+ if recipefile:
+ targets = get_bbclassextend_targets(recipefile, recipe)
+ if pn in targets:
+ workspacepn = recipe
+ break
+ else:
+ raise DevtoolError("No recipe named '%s' in your workspace" % pn)
+
+ if checksrc:
+ srctree = workspace[workspacepn]['srctree']
+ if not os.path.exists(srctree):
+ raise DevtoolError("Source tree %s for recipe %s does not exist" % (srctree, workspacepn))
+ if not os.listdir(srctree):
+ raise DevtoolError("Source tree %s for recipe %s is empty" % (srctree, workspacepn))
+
+ return workspacepn
+
+def use_external_build(same_dir, no_same_dir, d):
+ """
+ Determine if we should use B!=S (separate build and source directories) or not
+ """
+ b_is_s = True
+ if no_same_dir:
+ logger.info('Using separate build directory since --no-same-dir specified')
+ b_is_s = False
+ elif same_dir:
+ logger.info('Using source tree as build directory since --same-dir specified')
+ elif bb.data.inherits_class('autotools-brokensep', d):
+ logger.info('Using source tree as build directory since recipe inherits autotools-brokensep')
+ elif d.getVar('B') == os.path.abspath(d.getVar('S')):
+ logger.info('Using source tree as build directory since that would be the default for this recipe')
+ else:
+ b_is_s = False
+ return b_is_s
+
+def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
+ """
+ Set up the git repository for the source tree
+ """
+ import bb.process
+ import oe.patch
+ if not os.path.exists(os.path.join(repodir, '.git')):
+ bb.process.run('git init', cwd=repodir)
+ bb.process.run('git add .', cwd=repodir)
+ commit_cmd = ['git']
+ oe.patch.GitApplyTree.gitCommandUserOptions(commit_cmd, d=d)
+ commit_cmd += ['commit', '-q']
+ stdout, _ = bb.process.run('git status --porcelain', cwd=repodir)
+ if not stdout:
+ commit_cmd.append('--allow-empty')
+ commitmsg = "Initial empty commit with no upstream sources"
+ elif version:
+ commitmsg = "Initial commit from upstream at version %s" % version
+ else:
+ commitmsg = "Initial commit from upstream"
+ commit_cmd += ['-m', commitmsg]
+ bb.process.run(commit_cmd, cwd=repodir)
+
+ bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
+ bb.process.run('git tag -f %s' % basetag, cwd=repodir)
+
+def recipe_to_append(recipefile, config, wildcard=False):
+ """
+ Convert a recipe file to a bbappend file path within the workspace.
+ NOTE: if the bbappend already exists, you should be using
+ workspace[args.recipename]['bbappend'] instead of calling this
+ function.
+ """
+ appendname = os.path.splitext(os.path.basename(recipefile))[0]
+ if wildcard:
+ appendname = re.sub(r'_.*', '_%', appendname)
+ appendpath = os.path.join(config.workspace_path, 'appends')
+ appendfile = os.path.join(appendpath, appendname + '.bbappend')
+ return appendfile
+
+def get_bbclassextend_targets(recipefile, pn):
+ """
+ Cheap function to get BBCLASSEXTEND and then convert that to the
+ list of targets that would result.
+ """
+ import bb.utils
+
+ values = {}
+ def get_bbclassextend_varfunc(varname, origvalue, op, newlines):
+ values[varname] = origvalue
+ return origvalue, None, 0, True
+ with open(recipefile, 'r') as f:
+ bb.utils.edit_metadata(f, ['BBCLASSEXTEND'], get_bbclassextend_varfunc)
+
+ targets = []
+ bbclassextend = values.get('BBCLASSEXTEND', '').split()
+ if bbclassextend:
+ for variant in bbclassextend:
+ if variant == 'nativesdk':
+ targets.append('%s-%s' % (variant, pn))
+ elif variant in ['native', 'cross', 'crosssdk']:
+ targets.append('%s-%s' % (pn, variant))
+ return targets
+
+def ensure_npm(config, basepath, fixed_setup=False, check_exists=True):
+ """
+ Ensure that npm is available and either build it or show a
+ reasonable error message
+ """
+ if check_exists:
+ tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
+ try:
+ rd = tinfoil.parse_recipe('nodejs-native')
+ nativepath = rd.getVar('STAGING_BINDIR_NATIVE')
+ finally:
+ tinfoil.shutdown()
+ npmpath = os.path.join(nativepath, 'npm')
+ build_npm = not os.path.exists(npmpath)
+ else:
+ build_npm = True
+
+ if build_npm:
+ logger.info('Building nodejs-native')
+ try:
+ exec_build_env_command(config.init_path, basepath,
+ 'bitbake -q nodejs-native -c addto_recipe_sysroot', watch=True)
+ except bb.process.ExecutionError as e:
+ if "Nothing PROVIDES 'nodejs-native'" in e.stdout:
+ if fixed_setup:
+ msg = 'nodejs-native is required for npm but is not available within this SDK'
+ else:
+ msg = 'nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs'
+ raise DevtoolError(msg)
+ else:
+ raise
diff --git a/scripts/lib/devtool/build.py b/scripts/lib/devtool/build.py
new file mode 100644
index 0000000000..252379e9b2
--- /dev/null
+++ b/scripts/lib/devtool/build.py
@@ -0,0 +1,86 @@
+# Development tool - build command plugin
+#
+# Copyright (C) 2014-2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""Devtool build plugin"""
+
+import os
+import bb
+import logging
+import argparse
+import tempfile
+from devtool import exec_build_env_command, check_workspace_recipe, DevtoolError
+
+logger = logging.getLogger('devtool')
+
+
+def _set_file_values(fn, values):
+ remaining = list(values.keys())
+
+ def varfunc(varname, origvalue, op, newlines):
+ newvalue = values.get(varname, origvalue)
+ remaining.remove(varname)
+ return (newvalue, '=', 0, True)
+
+ with open(fn, 'r') as f:
+ (updated, newlines) = bb.utils.edit_metadata(f, values, varfunc)
+
+ for item in remaining:
+ updated = True
+ newlines.append('%s = "%s"' % (item, values[item]))
+
+ if updated:
+ with open(fn, 'w') as f:
+ f.writelines(newlines)
+ return updated
+
+def _get_build_tasks(config):
+ tasks = config.get('Build', 'build_task', 'populate_sysroot,packagedata').split(',')
+ return ['do_%s' % task.strip() for task in tasks]
+
+def build(args, config, basepath, workspace):
+ """Entry point for the devtool 'build' subcommand"""
+ workspacepn = check_workspace_recipe(workspace, args.recipename, bbclassextend=True)
+
+ build_tasks = _get_build_tasks(config)
+
+ bbappend = workspace[workspacepn]['bbappend']
+ if args.disable_parallel_make:
+ logger.info("Disabling 'make' parallelism")
+ _set_file_values(bbappend, {'PARALLEL_MAKE': ''})
+ try:
+ bbargs = []
+ for task in build_tasks:
+ if args.recipename.endswith('-native') and 'package' in task:
+ continue
+ bbargs.append('%s:%s' % (args.recipename, task))
+ exec_build_env_command(config.init_path, basepath, 'bitbake %s' % ' '.join(bbargs), watch=True)
+ except bb.process.ExecutionError as e:
+ # We've already seen the output since watch=True, so just ensure we return something to the user
+ return e.exitcode
+ finally:
+ if args.disable_parallel_make:
+ _set_file_values(bbappend, {'PARALLEL_MAKE': None})
+
+ return 0
+
+def register_commands(subparsers, context):
+ """Register devtool subcommands from this plugin"""
+ parser_build = subparsers.add_parser('build', help='Build a recipe',
+ description='Builds the specified recipe using bitbake (up to and including %s)' % ', '.join(_get_build_tasks(context.config)),
+ group='working', order=50)
+ parser_build.add_argument('recipename', help='Recipe to build')
+ parser_build.add_argument('-s', '--disable-parallel-make', action="store_true", help='Disable make parallelism')
+ parser_build.set_defaults(func=build)
diff --git a/scripts/lib/devtool/build_image.py b/scripts/lib/devtool/build_image.py
new file mode 100644
index 0000000000..e5810389be
--- /dev/null
+++ b/scripts/lib/devtool/build_image.py
@@ -0,0 +1,174 @@
+# Development tool - build-image plugin
+#
+# Copyright (C) 2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+"""Devtool plugin containing the build-image subcommand."""
+
+import os
+import errno
+import logging
+
+from bb.process import ExecutionError
+from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError
+
+logger = logging.getLogger('devtool')
+
+class TargetNotImageError(Exception):
+ pass
+
+def _get_packages(tinfoil, workspace, config):
+ """Get list of packages from recipes in the workspace."""
+ result = []
+ for recipe in workspace:
+ data = parse_recipe(config, tinfoil, recipe, True)
+ if 'class-target' in data.getVar('OVERRIDES').split(':'):
+ if recipe in data.getVar('PACKAGES').split():
+ result.append(recipe)
+ else:
+ logger.warning("Skipping recipe %s as it doesn't produce a "
+ "package with the same name", recipe)
+ return result
+
+def build_image(args, config, basepath, workspace):
+ """Entry point for the devtool 'build-image' subcommand."""
+
+ image = args.imagename
+ auto_image = False
+ if not image:
+ sdk_targets = config.get('SDK', 'sdk_targets', '').split()
+ if sdk_targets:
+ image = sdk_targets[0]
+ auto_image = True
+ if not image:
+ raise DevtoolError('Unable to determine image to build, please specify one')
+
+ try:
+ if args.add_packages:
+ add_packages = args.add_packages.split(',')
+ else:
+ add_packages = None
+ result, outputdir = build_image_task(config, basepath, workspace, image, add_packages)
+ except TargetNotImageError:
+ if auto_image:
+ raise DevtoolError('Unable to determine image to build, please specify one')
+ else:
+ raise DevtoolError('Specified recipe %s is not an image recipe' % image)
+
+ if result == 0:
+ logger.info('Successfully built %s. You can find output files in %s'
+ % (image, outputdir))
+ return result
+
+def build_image_task(config, basepath, workspace, image, add_packages=None, task=None, extra_append=None):
+ # remove <image>.bbappend to make sure setup_tinfoil doesn't
+ # break because of it
+ target_basename = config.get('SDK', 'target_basename', '')
+ if target_basename:
+ appendfile = os.path.join(config.workspace_path, 'appends',
+ '%s.bbappend' % target_basename)
+ try:
+ os.unlink(appendfile)
+ except OSError as exc:
+ if exc.errno != errno.ENOENT:
+ raise
+
+ tinfoil = setup_tinfoil(basepath=basepath)
+ try:
+ rd = parse_recipe(config, tinfoil, image, True)
+ if not rd:
+ # Error already shown
+ return (1, None)
+ if not bb.data.inherits_class('image', rd):
+ raise TargetNotImageError()
+
+ # Get the actual filename used and strip the .bb and full path
+ target_basename = rd.getVar('FILE')
+ target_basename = os.path.splitext(os.path.basename(target_basename))[0]
+ config.set('SDK', 'target_basename', target_basename)
+ config.write()
+
+ appendfile = os.path.join(config.workspace_path, 'appends',
+ '%s.bbappend' % target_basename)
+
+ outputdir = None
+ try:
+ if workspace or add_packages:
+ if add_packages:
+ packages = add_packages
+ else:
+ packages = _get_packages(tinfoil, workspace, config)
+ else:
+ packages = None
+ if not task:
+ if not packages and not add_packages and workspace:
+ logger.warning('No recipes in workspace, building image %s unmodified', image)
+ elif not packages:
+ logger.warning('No packages to add, building image %s unmodified', image)
+
+ if packages or extra_append:
+ bb.utils.mkdirhier(os.path.dirname(appendfile))
+ with open(appendfile, 'w') as afile:
+ if packages:
+ # include packages from workspace recipes into the image
+ afile.write('IMAGE_INSTALL_append = " %s"\n' % ' '.join(packages))
+ if not task:
+ logger.info('Building image %s with the following '
+ 'additional packages: %s', image, ' '.join(packages))
+ if extra_append:
+ for line in extra_append:
+ afile.write('%s\n' % line)
+
+ if task in ['populate_sdk', 'populate_sdk_ext']:
+ outputdir = rd.getVar('SDK_DEPLOY')
+ else:
+ outputdir = rd.getVar('DEPLOY_DIR_IMAGE')
+
+ tmp_tinfoil = tinfoil
+ tinfoil = None
+ tmp_tinfoil.shutdown()
+
+ options = ''
+ if task:
+ options += '-c %s' % task
+
+ # run bitbake to build image (or specified task)
+ try:
+ exec_build_env_command(config.init_path, basepath,
+ 'bitbake %s %s' % (options, image), watch=True)
+ except ExecutionError as err:
+ return (err.exitcode, None)
+ finally:
+ if os.path.isfile(appendfile):
+ os.unlink(appendfile)
+ finally:
+ if tinfoil:
+ tinfoil.shutdown()
+ return (0, outputdir)
+
+
+def register_commands(subparsers, context):
+ """Register devtool subcommands from the build-image plugin"""
+ parser = subparsers.add_parser('build-image',
+ help='Build image including workspace recipe packages',
+ description='Builds an image, extending it to include '
+ 'packages from recipes in the workspace',
+ group='testbuild', order=-10)
+ parser.add_argument('imagename', help='Image recipe to build', nargs='?')
+ parser.add_argument('-p', '--add-packages', help='Instead of adding packages for the '
+ 'entire workspace, specify packages to be added to the image '
+ '(separate multiple packages by commas)',
+ metavar='PACKAGES')
+ parser.set_defaults(func=build_image)
diff --git a/scripts/lib/devtool/build_sdk.py b/scripts/lib/devtool/build_sdk.py
new file mode 100644
index 0000000000..b89d65b0cb
--- /dev/null
+++ b/scripts/lib/devtool/build_sdk.py
@@ -0,0 +1,65 @@
+# Development tool - build-sdk command plugin
+#
+# Copyright (C) 2015-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import subprocess
+import logging
+import glob
+import shutil
+import errno
+import sys
+import tempfile
+from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError
+from devtool import build_image
+
+logger = logging.getLogger('devtool')
+
+
+def build_sdk(args, config, basepath, workspace):
+ """Entry point for the devtool build-sdk command"""
+
+ sdk_targets = config.get('SDK', 'sdk_targets', '').split()
+ if sdk_targets:
+ image = sdk_targets[0]
+ else:
+ raise DevtoolError('Unable to determine image to build SDK for')
+
+ extra_append = ['SDK_DERIVATIVE = "1"']
+ try:
+ result, outputdir = build_image.build_image_task(config,
+ basepath,
+ workspace,
+ image,
+ task='populate_sdk_ext',
+ extra_append=extra_append)
+ except build_image.TargetNotImageError:
+ raise DevtoolError('Unable to determine image to build SDK for')
+
+ if result == 0:
+ logger.info('Successfully built SDK. You can find output files in %s'
+ % outputdir)
+ return result
+
+
+def register_commands(subparsers, context):
+ """Register devtool subcommands"""
+ if context.fixed_setup:
+ parser_build_sdk = subparsers.add_parser('build-sdk',
+ help='Build a derivative SDK of this one',
+ description='Builds an extensible SDK based upon this one and the items in your workspace',
+ group='advanced')
+ parser_build_sdk.set_defaults(func=build_sdk)
diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py
index f016b23dba..b3730ae833 100644
--- a/scripts/lib/devtool/deploy.py
+++ b/scripts/lib/devtool/deploy.py
@@ -1,6 +1,6 @@
# Development tool - deploy/undeploy command plugin
#
-# Copyright (C) 2014-2015 Intel Corporation
+# Copyright (C) 2014-2016 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -14,92 +14,244 @@
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""Devtool plugin containing the deploy subcommands"""
import os
import subprocess
import logging
-from devtool import exec_build_env_command
+import tempfile
+import shutil
+import argparse_oe
+from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError
logger = logging.getLogger('devtool')
-def plugin_init(pluginlist):
- pass
+deploylist_path = '/.devtool'
+
+def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False):
+ """
+ Prepare a shell script for running on the target to
+ deploy/undeploy files. We have to be careful what we put in this
+ script - only commands that are likely to be available on the
+ target are suitable (the target might be constrained, e.g. using
+ busybox rather than bash with coreutils).
+ """
+ lines = []
+ lines.append('#!/bin/sh')
+ lines.append('set -e')
+ if undeployall:
+ # Yes, I know this is crude - but it does work
+ lines.append('for entry in %s/*.list; do' % deploylist_path)
+ lines.append('[ ! -f $entry ] && exit')
+ lines.append('set `basename $entry | sed "s/.list//"`')
+ if dryrun:
+ if not deploy:
+ lines.append('echo "Previously deployed files for $1:"')
+ lines.append('manifest="%s/$1.list"' % deploylist_path)
+ lines.append('preservedir="%s/$1.preserve"' % deploylist_path)
+ lines.append('if [ -f $manifest ] ; then')
+ # Read manifest in reverse and delete files / remove empty dirs
+ lines.append(' sed \'1!G;h;$!d\' $manifest | while read file')
+ lines.append(' do')
+ if dryrun:
+ lines.append(' if [ ! -d $file ] ; then')
+ lines.append(' echo $file')
+ lines.append(' fi')
+ else:
+ lines.append(' if [ -d $file ] ; then')
+ # Avoid deleting a preserved directory in case it has special perms
+ lines.append(' if [ ! -d $preservedir/$file ] ; then')
+ lines.append(' rmdir $file > /dev/null 2>&1 || true')
+ lines.append(' fi')
+ lines.append(' else')
+ lines.append(' rm $file')
+ lines.append(' fi')
+ lines.append(' done')
+ if not dryrun:
+ lines.append(' rm $manifest')
+ if not deploy and not dryrun:
+ # May as well remove all traces
+ lines.append(' rmdir `dirname $manifest` > /dev/null 2>&1 || true')
+ lines.append('fi')
+
+ if deploy:
+ if not nocheckspace:
+ # Check for available space
+ # FIXME This doesn't take into account files spread across multiple
+ # partitions, but doing that is non-trivial
+ # Find the part of the destination path that exists
+ lines.append('checkpath="$2"')
+ lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]')
+ lines.append('do')
+ lines.append(' checkpath=`dirname "$checkpath"`')
+ lines.append('done')
+ lines.append(r'freespace=$(df -P $checkpath | sed -nre "s/^(\S+\s+){3}([0-9]+).*/\2/p")')
+ # First line of the file is the total space
+ lines.append('total=`head -n1 $3`')
+ lines.append('if [ $total -gt $freespace ] ; then')
+ lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"')
+ lines.append(' exit 1')
+ lines.append('fi')
+ if not nopreserve:
+ # Preserve any files that exist. Note that this will add to the
+ # preserved list with successive deployments if the list of files
+ # deployed changes, but because we've deleted any previously
+ # deployed files at this point it will never preserve anything
+ # that was deployed, only files that existed prior to any deploying
+ # (which makes the most sense)
+ lines.append('cat $3 | sed "1d" | while read file fsize')
+ lines.append('do')
+ lines.append(' if [ -e $file ] ; then')
+ lines.append(' dest="$preservedir/$file"')
+ lines.append(' mkdir -p `dirname $dest`')
+ lines.append(' mv $file $dest')
+ lines.append(' fi')
+ lines.append('done')
+ lines.append('rm $3')
+ lines.append('mkdir -p `dirname $manifest`')
+ lines.append('mkdir -p $2')
+ if verbose:
+ lines.append(' tar xv -C $2 -f - | tee $manifest')
+ else:
+ lines.append(' tar xv -C $2 -f - > $manifest')
+ lines.append('sed -i "s!^./!$2!" $manifest')
+ elif not dryrun:
+ # Put any preserved files back
+ lines.append('if [ -d $preservedir ] ; then')
+ lines.append(' cd $preservedir')
+ lines.append(' find . -type f -exec mv {} /{} \;')
+ lines.append(' cd /')
+ lines.append(' rm -rf $preservedir')
+ lines.append('fi')
+
+ if undeployall:
+ if not dryrun:
+ lines.append('echo "NOTE: Successfully undeployed $1"')
+ lines.append('done')
+
+ # Delete the script itself
+ lines.append('rm $0')
+ lines.append('')
+
+ return '\n'.join(lines)
def deploy(args, config, basepath, workspace):
+ """Entry point for the devtool 'deploy' subcommand"""
import re
- from devtool import exec_build_env_command
+ import math
+ import oe.recipeutils
+
+ check_workspace_recipe(workspace, args.recipename, checksrc=False)
- if not args.recipename in workspace:
- logger.error("no recipe named %s in your workspace" % args.recipename)
- return -1
try:
host, destdir = args.target.split(':')
except ValueError:
destdir = '/'
else:
args.target = host
+ if not destdir.endswith('/'):
+ destdir += '/'
- deploy_dir = os.path.join(basepath, 'target_deploy', args.target)
- deploy_file = os.path.join(deploy_dir, args.recipename + '.list')
-
- stdout, stderr = exec_build_env_command(config.init_path, basepath, 'bitbake -e %s' % args.recipename, shell=True)
- recipe_outdir = re.search(r'^D="(.*)"', stdout, re.MULTILINE).group(1)
- if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
- logger.error('No files to deploy - have you built the %s recipe? If so, the install step has not installed any files.' % args.recipename)
- return -1
+ tinfoil = setup_tinfoil(basepath=basepath)
+ try:
+ try:
+ rd = tinfoil.parse_recipe(args.recipename)
+ except Exception as e:
+ raise DevtoolError('Exception parsing recipe %s: %s' %
+ (args.recipename, e))
+ recipe_outdir = rd.getVar('D')
+ if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
+ raise DevtoolError('No files to deploy - have you built the %s '
+ 'recipe? If so, the install step has not installed '
+ 'any files.' % args.recipename)
- if args.dry_run:
- print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
- for root, dirs, files in os.walk(recipe_outdir):
+ filelist = []
+ ftotalsize = 0
+ for root, _, files in os.walk(recipe_outdir):
for fn in files:
- print(' %s' % os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn))
- return 0
+ # Get the size in kiB (since we'll be comparing it to the output of du -k)
+ # MUST use lstat() here not stat() or getfilesize() since we don't want to
+ # dereference symlinks
+ fsize = int(math.ceil(float(os.lstat(os.path.join(root, fn)).st_size)/1024))
+ ftotalsize += fsize
+ # The path as it would appear on the target
+ fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
+ filelist.append((fpath, fsize))
- if os.path.exists(deploy_file):
- if undeploy(args, config, basepath, workspace):
- # Error already shown
- return -1
+ if args.dry_run:
+ print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
+ for item, _ in filelist:
+ print(' %s' % item)
+ return 0
- extraoptions = ''
- if args.no_host_check:
- extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
- if not args.show_status:
- extraoptions += ' -q'
- ret = subprocess.call('scp -r %s %s/* %s:%s' % (extraoptions, recipe_outdir, args.target, destdir), shell=True)
- if ret != 0:
- logger.error('Deploy failed - rerun with -s to get a complete error message')
- return ret
- logger.info('Successfully deployed %s' % recipe_outdir)
+ extraoptions = ''
+ if args.no_host_check:
+ extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+ if not args.show_status:
+ extraoptions += ' -q'
- if not os.path.exists(deploy_dir):
- os.makedirs(deploy_dir)
+ scp_port = ''
+ ssh_port = ''
+ if not args.port:
+ raise DevtoolError("If you specify -P/--port then you must provide the port to be used to connect to the target")
+ else:
+ scp_port = "-P %s" % args.port
+ ssh_port = "-p %s" % args.port
- files_list = []
- for root, _, files in os.walk(recipe_outdir):
- for filename in files:
- filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
- files_list.append(os.path.join(destdir, filename))
+ # In order to delete previously deployed files and have the manifest file on
+ # the target, we write out a shell script and then copy it to the target
+ # so we can then run it (piping tar output to it).
+ # (We cannot use scp here, because it doesn't preserve symlinks.)
+ tmpdir = tempfile.mkdtemp(prefix='devtool')
+ try:
+ tmpscript = '/tmp/devtool_deploy.sh'
+ tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
+ shellscript = _prepare_remote_script(deploy=True,
+ verbose=args.show_status,
+ nopreserve=args.no_preserve,
+ nocheckspace=args.no_check_space)
+ # Write out the script to a file
+ with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
+ f.write(shellscript)
+ # Write out the file list
+ with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
+ f.write('%d\n' % ftotalsize)
+ for fpath, fsize in filelist:
+ f.write('%s %d\n' % (fpath, fsize))
+ # Copy them to the target
+ ret = subprocess.call("scp %s %s %s/* %s:%s" % (scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
+ if ret != 0:
+ raise DevtoolError('Failed to copy script to %s - rerun with -s to '
+ 'get a complete error message' % args.target)
+ finally:
+ shutil.rmtree(tmpdir)
- with open(deploy_file, 'w') as fobj:
- fobj.write('\n'.join(files_list))
+ # Now run the script
+ ret = exec_fakeroot(rd, 'tar cf - . | ssh %s %s %s \'sh %s %s %s %s\'' % (ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True)
+ if ret != 0:
+ raise DevtoolError('Deploy failed - rerun with -s to get a complete '
+ 'error message')
- return 0
+ logger.info('Successfully deployed %s' % recipe_outdir)
-def undeploy(args, config, basepath, workspace):
+ files_list = []
+ for root, _, files in os.walk(recipe_outdir):
+ for filename in files:
+ filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
+ files_list.append(os.path.join(destdir, filename))
+ finally:
+ tinfoil.shutdown()
- deploy_file = os.path.join(basepath, 'target_deploy', args.target, args.recipename + '.list')
- if not os.path.exists(deploy_file):
- logger.error('%s has not been deployed' % args.recipename)
- return -1
+ return 0
- if args.dry_run:
- print('Previously deployed files to be un-deployed for %s on target %s:' % (args.recipename, args.target))
- with open(deploy_file, 'r') as f:
- for line in f:
- print(' %s' % line.rstrip())
- return 0
+def undeploy(args, config, basepath, workspace):
+ """Entry point for the devtool 'undeploy' subcommand"""
+ if args.all and args.recipename:
+ raise argparse_oe.ArgumentUsageError('Cannot specify -a/--all with a recipe name', 'undeploy-target')
+ elif not args.recipename and not args.all:
+ raise argparse_oe.ArgumentUsageError('If you don\'t specify a recipe, you must specify -a/--all', 'undeploy-target')
extraoptions = ''
if args.no_host_check:
@@ -107,34 +259,67 @@ def undeploy(args, config, basepath, workspace):
if not args.show_status:
extraoptions += ' -q'
- ret = subprocess.call("scp %s %s %s:/tmp" % (extraoptions, deploy_file, args.target), shell=True)
+ scp_port = ''
+ ssh_port = ''
+ if not args.port:
+ raise DevtoolError("If you specify -P/--port then you must provide the port to be used to connect to the target")
+ else:
+ scp_port = "-P %s" % args.port
+ ssh_port = "-p %s" % args.port
+
+ args.target = args.target.split(':')[0]
+
+ tmpdir = tempfile.mkdtemp(prefix='devtool')
+ try:
+ tmpscript = '/tmp/devtool_undeploy.sh'
+ shellscript = _prepare_remote_script(deploy=False, dryrun=args.dry_run, undeployall=args.all)
+ # Write out the script to a file
+ with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
+ f.write(shellscript)
+ # Copy it to the target
+ ret = subprocess.call("scp %s %s %s/* %s:%s" % (scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
+ if ret != 0:
+ raise DevtoolError('Failed to copy script to %s - rerun with -s to '
+ 'get a complete error message' % args.target)
+ finally:
+ shutil.rmtree(tmpdir)
+
+ # Now run the script
+ ret = subprocess.call('ssh %s %s %s \'sh %s %s\'' % (ssh_port, extraoptions, args.target, tmpscript, args.recipename), shell=True)
if ret != 0:
- logger.error('Failed to copy file list to %s - rerun with -s to get a complete error message' % args.target)
- return -1
+ raise DevtoolError('Undeploy failed - rerun with -s to get a complete '
+ 'error message')
- ret = subprocess.call("ssh %s %s 'xargs -n1 rm -f </tmp/%s'" % (extraoptions, args.target, os.path.basename(deploy_file)), shell=True)
- if ret == 0:
+ if not args.all and not args.dry_run:
logger.info('Successfully undeployed %s' % args.recipename)
- os.remove(deploy_file)
- else:
- logger.error('Undeploy failed - rerun with -s to get a complete error message')
-
- return ret
+ return 0
def register_commands(subparsers, context):
- parser_deploy = subparsers.add_parser('deploy-target', help='Deploy recipe output files to live target machine')
+ """Register devtool subcommands from the deploy plugin"""
+ parser_deploy = subparsers.add_parser('deploy-target',
+ help='Deploy recipe output files to live target machine',
+ description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.',
+ group='testbuild')
parser_deploy.add_argument('recipename', help='Recipe to deploy')
parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true')
+ parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true')
+ parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true')
+ parser_deploy.add_argument('-P', '--port', default='22', help='Port to use for connection to the target')
parser_deploy.set_defaults(func=deploy)
- parser_undeploy = subparsers.add_parser('undeploy-target', help='Undeploy recipe output files in live target machine')
- parser_undeploy.add_argument('recipename', help='Recipe to undeploy')
+ parser_undeploy = subparsers.add_parser('undeploy-target',
+ help='Undeploy recipe output files in live target machine',
+ description='Un-deploys recipe output files previously deployed to a live target machine by devtool deploy-target.',
+ group='testbuild')
+ parser_undeploy.add_argument('recipename', help='Recipe to undeploy (if not using -a/--all)', nargs='?')
parser_undeploy.add_argument('target', help='Live target machine running an ssh server: user@hostname')
parser_undeploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
parser_undeploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
+ parser_undeploy.add_argument('-a', '--all', help='Undeploy all recipes deployed on the target', action='store_true')
parser_undeploy.add_argument('-n', '--dry-run', help='List files to be undeployed only', action='store_true')
+ parser_undeploy.add_argument('-P', '--port', default='22', help='Port to use for connection to the target')
parser_undeploy.set_defaults(func=undeploy)
diff --git a/scripts/lib/devtool/package.py b/scripts/lib/devtool/package.py
new file mode 100644
index 0000000000..af9e8f15f5
--- /dev/null
+++ b/scripts/lib/devtool/package.py
@@ -0,0 +1,60 @@
+# Development tool - package command plugin
+#
+# Copyright (C) 2014-2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""Devtool plugin containing the package subcommands"""
+
+import os
+import subprocess
+import logging
+from bb.process import ExecutionError
+from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, DevtoolError
+
+logger = logging.getLogger('devtool')
+
+def package(args, config, basepath, workspace):
+ """Entry point for the devtool 'package' subcommand"""
+ check_workspace_recipe(workspace, args.recipename)
+
+ tinfoil = setup_tinfoil(basepath=basepath, config_only=True)
+ try:
+ image_pkgtype = config.get('Package', 'image_pkgtype', '')
+ if not image_pkgtype:
+ image_pkgtype = tinfoil.config_data.getVar('IMAGE_PKGTYPE')
+
+ deploy_dir_pkg = tinfoil.config_data.getVar('DEPLOY_DIR_%s' % image_pkgtype.upper())
+ finally:
+ tinfoil.shutdown()
+
+ package_task = config.get('Package', 'package_task', 'package_write_%s' % image_pkgtype)
+ try:
+ exec_build_env_command(config.init_path, basepath, 'bitbake -c %s %s' % (package_task, args.recipename), watch=True)
+ except bb.process.ExecutionError as e:
+ # We've already seen the output since watch=True, so just ensure we return something to the user
+ return e.exitcode
+
+ logger.info('Your packages are in %s' % deploy_dir_pkg)
+
+ return 0
+
+def register_commands(subparsers, context):
+ """Register devtool subcommands from the package plugin"""
+ if context.fixed_setup:
+ parser_package = subparsers.add_parser('package',
+ help='Build packages for a recipe',
+ description='Builds packages for a recipe\'s output files',
+ group='testbuild', order=-5)
+ parser_package.add_argument('recipename', help='Recipe to package')
+ parser_package.set_defaults(func=package)
diff --git a/scripts/lib/devtool/runqemu.py b/scripts/lib/devtool/runqemu.py
new file mode 100644
index 0000000000..e26cf28c2f
--- /dev/null
+++ b/scripts/lib/devtool/runqemu.py
@@ -0,0 +1,74 @@
+# Development tool - runqemu command plugin
+#
+# Copyright (C) 2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+"""Devtool runqemu plugin"""
+
+import os
+import bb
+import logging
+import argparse
+import glob
+from devtool import exec_build_env_command, setup_tinfoil, DevtoolError
+
+logger = logging.getLogger('devtool')
+
+def runqemu(args, config, basepath, workspace):
+ """Entry point for the devtool 'runqemu' subcommand"""
+
+ tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
+ try:
+ machine = tinfoil.config_data.getVar('MACHINE')
+ bindir_native = os.path.join(tinfoil.config_data.getVar('STAGING_DIR'),
+ tinfoil.config_data.getVar('BUILD_ARCH'),
+ tinfoil.config_data.getVar('bindir_native').lstrip(os.path.sep))
+ finally:
+ tinfoil.shutdown()
+
+ if not glob.glob(os.path.join(bindir_native, 'qemu-system-*')):
+ raise DevtoolError('QEMU is not available within this SDK')
+
+ imagename = args.imagename
+ if not imagename:
+ sdk_targets = config.get('SDK', 'sdk_targets', '').split()
+ if sdk_targets:
+ imagename = sdk_targets[0]
+ if not imagename:
+ raise DevtoolError('Unable to determine image name to run, please specify one')
+
+ try:
+ # FIXME runqemu assumes that if OECORE_NATIVE_SYSROOT is set then it shouldn't
+ # run bitbake to find out the values of various environment variables, which
+ # isn't the case for the extensible SDK. Work around it for now.
+ newenv = dict(os.environ)
+ newenv.pop('OECORE_NATIVE_SYSROOT', '')
+ exec_build_env_command(config.init_path, basepath, 'runqemu %s %s %s' % (machine, imagename, " ".join(args.args)), watch=True, env=newenv)
+ except bb.process.ExecutionError as e:
+ # We've already seen the output since watch=True, so just ensure we return something to the user
+ return e.exitcode
+
+ return 0
+
+def register_commands(subparsers, context):
+ """Register devtool subcommands from this plugin"""
+ if context.fixed_setup:
+ parser_runqemu = subparsers.add_parser('runqemu', help='Run QEMU on the specified image',
+ description='Runs QEMU to boot the specified image',
+ group='testbuild', order=-20)
+ parser_runqemu.add_argument('imagename', help='Name of built image to boot within QEMU', nargs='?')
+ parser_runqemu.add_argument('args', help='Any remaining arguments are passed to the runqemu script (pass --help after imagename to see what these are)',
+ nargs=argparse.REMAINDER)
+ parser_runqemu.set_defaults(func=runqemu)
diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py
new file mode 100644
index 0000000000..e8bf0ad98c
--- /dev/null
+++ b/scripts/lib/devtool/sdk.py
@@ -0,0 +1,336 @@
+# Development tool - sdk-update command plugin
+#
+# Copyright (C) 2015-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import subprocess
+import logging
+import glob
+import shutil
+import errno
+import sys
+import tempfile
+import re
+from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError
+
+logger = logging.getLogger('devtool')
+
+def parse_locked_sigs(sigfile_path):
+ """Return <pn:task>:<hash> dictionary"""
+ sig_dict = {}
+ with open(sigfile_path) as f:
+ lines = f.readlines()
+ for line in lines:
+ if ':' in line:
+ taskkey, _, hashval = line.rpartition(':')
+ sig_dict[taskkey.strip()] = hashval.split()[0]
+ return sig_dict
+
+def generate_update_dict(sigfile_new, sigfile_old):
+ """Return a dict containing <pn:task>:<hash> which indicates what need to be updated"""
+ update_dict = {}
+ sigdict_new = parse_locked_sigs(sigfile_new)
+ sigdict_old = parse_locked_sigs(sigfile_old)
+ for k in sigdict_new:
+ if k not in sigdict_old:
+ update_dict[k] = sigdict_new[k]
+ continue
+ if sigdict_new[k] != sigdict_old[k]:
+ update_dict[k] = sigdict_new[k]
+ continue
+ return update_dict
+
+def get_sstate_objects(update_dict, sstate_dir):
+ """Return a list containing sstate objects which are to be installed"""
+ sstate_objects = []
+ for k in update_dict:
+ files = set()
+ hashval = update_dict[k]
+ p = sstate_dir + '/' + hashval[:2] + '/*' + hashval + '*.tgz'
+ files |= set(glob.glob(p))
+ p = sstate_dir + '/*/' + hashval[:2] + '/*' + hashval + '*.tgz'
+ files |= set(glob.glob(p))
+ files = list(files)
+ if len(files) == 1:
+ sstate_objects.extend(files)
+ elif len(files) > 1:
+ logger.error("More than one matching sstate object found for %s" % hashval)
+
+ return sstate_objects
+
+def mkdir(d):
+ try:
+ os.makedirs(d)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise e
+
+def install_sstate_objects(sstate_objects, src_sdk, dest_sdk):
+ """Install sstate objects into destination SDK"""
+ sstate_dir = os.path.join(dest_sdk, 'sstate-cache')
+ if not os.path.exists(sstate_dir):
+ logger.error("Missing sstate-cache directory in %s, it might not be an extensible SDK." % dest_sdk)
+ raise
+ for sb in sstate_objects:
+ dst = sb.replace(src_sdk, dest_sdk)
+ destdir = os.path.dirname(dst)
+ mkdir(destdir)
+ logger.debug("Copying %s to %s" % (sb, dst))
+ shutil.copy(sb, dst)
+
+def check_manifest(fn, basepath):
+ import bb.utils
+ changedfiles = []
+ with open(fn, 'r') as f:
+ for line in f:
+ splitline = line.split()
+ if len(splitline) > 1:
+ chksum = splitline[0]
+ fpath = splitline[1]
+ curr_chksum = bb.utils.sha256_file(os.path.join(basepath, fpath))
+ if chksum != curr_chksum:
+ logger.debug('File %s changed: old csum = %s, new = %s' % (os.path.join(basepath, fpath), curr_chksum, chksum))
+ changedfiles.append(fpath)
+ return changedfiles
+
+def sdk_update(args, config, basepath, workspace):
+ """Entry point for devtool sdk-update command"""
+ updateserver = args.updateserver
+ if not updateserver:
+ updateserver = config.get('SDK', 'updateserver', '')
+ logger.debug("updateserver: %s" % updateserver)
+
+ # Make sure we are using sdk-update from within SDK
+ logger.debug("basepath = %s" % basepath)
+ old_locked_sig_file_path = os.path.join(basepath, 'conf/locked-sigs.inc')
+ if not os.path.exists(old_locked_sig_file_path):
+ logger.error("Not using devtool's sdk-update command from within an extensible SDK. Please specify correct basepath via --basepath option")
+ return -1
+ else:
+ logger.debug("Found conf/locked-sigs.inc in %s" % basepath)
+
+ if not '://' in updateserver:
+ logger.error("Update server must be a URL")
+ return -1
+
+ layers_dir = os.path.join(basepath, 'layers')
+ conf_dir = os.path.join(basepath, 'conf')
+
+ # Grab variable values
+ tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
+ try:
+ stamps_dir = tinfoil.config_data.getVar('STAMPS_DIR')
+ sstate_mirrors = tinfoil.config_data.getVar('SSTATE_MIRRORS')
+ site_conf_version = tinfoil.config_data.getVar('SITE_CONF_VERSION')
+ finally:
+ tinfoil.shutdown()
+
+ tmpsdk_dir = tempfile.mkdtemp()
+ try:
+ os.makedirs(os.path.join(tmpsdk_dir, 'conf'))
+ new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc')
+ # Fetch manifest from server
+ tmpmanifest = os.path.join(tmpsdk_dir, 'conf', 'sdk-conf-manifest')
+ ret = subprocess.call("wget -q -O %s %s/conf/sdk-conf-manifest" % (tmpmanifest, updateserver), shell=True)
+ changedfiles = check_manifest(tmpmanifest, basepath)
+ if not changedfiles:
+ logger.info("Already up-to-date")
+ return 0
+ # Update metadata
+ logger.debug("Updating metadata via git ...")
+ #Check for the status before doing a fetch and reset
+ if os.path.exists(os.path.join(basepath, 'layers/.git')):
+ out = subprocess.check_output("git status --porcelain", shell=True, cwd=layers_dir)
+ if not out:
+ ret = subprocess.call("git fetch --all; git reset --hard", shell=True, cwd=layers_dir)
+ else:
+ logger.error("Failed to update metadata as there have been changes made to it. Aborting.");
+ logger.error("Changed files:\n%s" % out);
+ return -1
+ else:
+ ret = -1
+ if ret != 0:
+ ret = subprocess.call("git clone %s/layers/.git" % updateserver, shell=True, cwd=tmpsdk_dir)
+ if ret != 0:
+ logger.error("Updating metadata via git failed")
+ return ret
+ logger.debug("Updating conf files ...")
+ for changedfile in changedfiles:
+ ret = subprocess.call("wget -q -O %s %s/%s" % (changedfile, updateserver, changedfile), shell=True, cwd=tmpsdk_dir)
+ if ret != 0:
+ logger.error("Updating %s failed" % changedfile)
+ return ret
+
+ # Check if UNINATIVE_CHECKSUM changed
+ uninative = False
+ if 'conf/local.conf' in changedfiles:
+ def read_uninative_checksums(fn):
+ chksumitems = []
+ with open(fn, 'r') as f:
+ for line in f:
+ if line.startswith('UNINATIVE_CHECKSUM'):
+ splitline = re.split(r'[\[\]"\']', line)
+ if len(splitline) > 3:
+ chksumitems.append((splitline[1], splitline[3]))
+ return chksumitems
+
+ oldsums = read_uninative_checksums(os.path.join(basepath, 'conf/local.conf'))
+ newsums = read_uninative_checksums(os.path.join(tmpsdk_dir, 'conf/local.conf'))
+ if oldsums != newsums:
+ uninative = True
+ for buildarch, chksum in newsums:
+ uninative_file = os.path.join('downloads', 'uninative', chksum, '%s-nativesdk-libc.tar.bz2' % buildarch)
+ mkdir(os.path.join(tmpsdk_dir, os.path.dirname(uninative_file)))
+ ret = subprocess.call("wget -q -O %s %s/%s" % (uninative_file, updateserver, uninative_file), shell=True, cwd=tmpsdk_dir)
+
+ # Ok, all is well at this point - move everything over
+ tmplayers_dir = os.path.join(tmpsdk_dir, 'layers')
+ if os.path.exists(tmplayers_dir):
+ shutil.rmtree(layers_dir)
+ shutil.move(tmplayers_dir, layers_dir)
+ for changedfile in changedfiles:
+ destfile = os.path.join(basepath, changedfile)
+ os.remove(destfile)
+ shutil.move(os.path.join(tmpsdk_dir, changedfile), destfile)
+ os.remove(os.path.join(conf_dir, 'sdk-conf-manifest'))
+ shutil.move(tmpmanifest, conf_dir)
+ if uninative:
+ shutil.rmtree(os.path.join(basepath, 'downloads', 'uninative'))
+ shutil.move(os.path.join(tmpsdk_dir, 'downloads', 'uninative'), os.path.join(basepath, 'downloads'))
+
+ if not sstate_mirrors:
+ with open(os.path.join(conf_dir, 'site.conf'), 'a') as f:
+ f.write('SCONF_VERSION = "%s"\n' % site_conf_version)
+ f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver)
+ finally:
+ shutil.rmtree(tmpsdk_dir)
+
+ if not args.skip_prepare:
+ # Find all potentially updateable tasks
+ sdk_update_targets = []
+ tasks = ['do_populate_sysroot', 'do_packagedata']
+ for root, _, files in os.walk(stamps_dir):
+ for fn in files:
+ if not '.sigdata.' in fn:
+ for task in tasks:
+ if '.%s.' % task in fn or '.%s_setscene.' % task in fn:
+ sdk_update_targets.append('%s:%s' % (os.path.basename(root), task))
+ # Run bitbake command for the whole SDK
+ logger.info("Preparing build system... (This may take some time.)")
+ try:
+ exec_build_env_command(config.init_path, basepath, 'bitbake --setscene-only %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT)
+ output, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -n %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT)
+ runlines = []
+ for line in output.splitlines():
+ if 'Running task ' in line:
+ runlines.append(line)
+ if runlines:
+ logger.error('Unexecuted tasks found in preparation log:\n %s' % '\n '.join(runlines))
+ return -1
+ except bb.process.ExecutionError as e:
+ logger.error('Preparation failed:\n%s' % e.stdout)
+ return -1
+ return 0
+
+def sdk_install(args, config, basepath, workspace):
+ """Entry point for the devtool sdk-install command"""
+
+ import oe.recipeutils
+ import bb.process
+
+ for recipe in args.recipename:
+ if recipe in workspace:
+ raise DevtoolError('recipe %s is a recipe in your workspace' % recipe)
+
+ tasks = ['do_populate_sysroot', 'do_packagedata']
+ stampprefixes = {}
+ def checkstamp(recipe):
+ stampprefix = stampprefixes[recipe]
+ stamps = glob.glob(stampprefix + '*')
+ for stamp in stamps:
+ if '.sigdata.' not in stamp and stamp.startswith((stampprefix + '.', stampprefix + '_setscene.')):
+ return True
+ else:
+ return False
+
+ install_recipes = []
+ tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
+ try:
+ for recipe in args.recipename:
+ rd = parse_recipe(config, tinfoil, recipe, True)
+ if not rd:
+ return 1
+ stampprefixes[recipe] = '%s.%s' % (rd.getVar('STAMP'), tasks[0])
+ if checkstamp(recipe):
+ logger.info('%s is already installed' % recipe)
+ else:
+ install_recipes.append(recipe)
+ finally:
+ tinfoil.shutdown()
+
+ if install_recipes:
+ logger.info('Installing %s...' % ', '.join(install_recipes))
+ install_tasks = []
+ for recipe in install_recipes:
+ for task in tasks:
+ if recipe.endswith('-native') and 'package' in task:
+ continue
+ install_tasks.append('%s:%s' % (recipe, task))
+ options = ''
+ if not args.allow_build:
+ options += ' --setscene-only'
+ try:
+ exec_build_env_command(config.init_path, basepath, 'bitbake %s %s' % (options, ' '.join(install_tasks)), watch=True)
+ except bb.process.ExecutionError as e:
+ raise DevtoolError('Failed to install %s:\n%s' % (recipe, str(e)))
+ failed = False
+ for recipe in install_recipes:
+ if checkstamp(recipe):
+ logger.info('Successfully installed %s' % recipe)
+ else:
+ raise DevtoolError('Failed to install %s - unavailable' % recipe)
+ failed = True
+ if failed:
+ return 2
+
+ try:
+ exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots', watch=True)
+ except bb.process.ExecutionError as e:
+ raise DevtoolError('Failed to bitbake build-sysroots:\n%s' % (str(e)))
+
+
+def register_commands(subparsers, context):
+ """Register devtool subcommands from the sdk plugin"""
+ if context.fixed_setup:
+ parser_sdk = subparsers.add_parser('sdk-update',
+ help='Update SDK components',
+ description='Updates installed SDK components from a remote server',
+ group='sdk')
+ updateserver = context.config.get('SDK', 'updateserver', '')
+ if updateserver:
+ parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from (default %s)' % updateserver, nargs='?')
+ else:
+ parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from')
+ parser_sdk.add_argument('--skip-prepare', action="store_true", help='Skip re-preparing the build system after updating (for debugging only)')
+ parser_sdk.set_defaults(func=sdk_update)
+
+ parser_sdk_install = subparsers.add_parser('sdk-install',
+ help='Install additional SDK components',
+ description='Installs additional recipe development files into the SDK. (You can use "devtool search" to find available recipes.)',
+ group='sdk')
+ parser_sdk_install.add_argument('recipename', help='Name of the recipe to install the development artifacts for', nargs='+')
+ parser_sdk_install.add_argument('-s', '--allow-build', help='Allow building requested item(s) from source', action='store_true')
+ parser_sdk_install.set_defaults(func=sdk_install)
diff --git a/scripts/lib/devtool/search.py b/scripts/lib/devtool/search.py
new file mode 100644
index 0000000000..054985b85d
--- /dev/null
+++ b/scripts/lib/devtool/search.py
@@ -0,0 +1,88 @@
+# Development tool - search command plugin
+#
+# Copyright (C) 2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+"""Devtool search plugin"""
+
+import os
+import bb
+import logging
+import argparse
+import re
+from devtool import setup_tinfoil, parse_recipe, DevtoolError
+
+logger = logging.getLogger('devtool')
+
+def search(args, config, basepath, workspace):
+ """Entry point for the devtool 'search' subcommand"""
+
+ tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
+ try:
+ pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
+ defsummary = tinfoil.config_data.getVar('SUMMARY', False) or ''
+
+ keyword_rc = re.compile(args.keyword)
+
+ for fn in os.listdir(pkgdata_dir):
+ pfn = os.path.join(pkgdata_dir, fn)
+ if not os.path.isfile(pfn):
+ continue
+
+ packages = []
+ match = False
+ if keyword_rc.search(fn):
+ match = True
+
+ if not match:
+ with open(pfn, 'r') as f:
+ for line in f:
+ if line.startswith('PACKAGES:'):
+ packages = line.split(':', 1)[1].strip().split()
+
+ for pkg in packages:
+ if keyword_rc.search(pkg):
+ match = True
+ break
+ if os.path.exists(os.path.join(pkgdata_dir, 'runtime', pkg + '.packaged')):
+ with open(os.path.join(pkgdata_dir, 'runtime', pkg), 'r') as f:
+ for line in f:
+ if ': ' in line:
+ splitline = line.split(':', 1)
+ key = splitline[0]
+ value = splitline[1].strip()
+ if key in ['PKG_%s' % pkg, 'DESCRIPTION', 'FILES_INFO'] or key.startswith('FILERPROVIDES_'):
+ if keyword_rc.search(value):
+ match = True
+ break
+
+ if match:
+ rd = parse_recipe(config, tinfoil, fn, True)
+ summary = rd.getVar('SUMMARY')
+ if summary == rd.expand(defsummary):
+ summary = ''
+ print("%s %s" % (fn.ljust(20), summary))
+ finally:
+ tinfoil.shutdown()
+
+ return 0
+
+def register_commands(subparsers, context):
+ """Register devtool subcommands from this plugin"""
+ parser_search = subparsers.add_parser('search', help='Search available recipes',
+ description='Searches for available target recipes. Matches on recipe name, package name, description and installed files, and prints the recipe name on match.',
+ group='info')
+ parser_search.add_argument('keyword', help='Keyword to search for (regular expression syntax allowed)')
+ parser_search.set_defaults(func=search, no_workspace=True)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index a459c7b57c..5ff1e230fd 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -1,6 +1,6 @@
# Development tool - standard commands plugin
#
-# Copyright (C) 2014-2015 Intel Corporation
+# Copyright (C) 2014-2016 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -14,277 +14,678 @@
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+"""Devtool standard plugins"""
import os
import sys
import re
import shutil
-import glob
+import subprocess
import tempfile
import logging
import argparse
+import argparse_oe
import scriptutils
-from devtool import exec_build_env_command, setup_tinfoil
+import errno
+import glob
+import filecmp
+from collections import OrderedDict
+from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, ensure_npm, DevtoolError
+from devtool import parse_recipe
logger = logging.getLogger('devtool')
-def plugin_init(pluginlist):
- pass
-
def add(args, config, basepath, workspace):
+ """Entry point for the devtool 'add' subcommand"""
import bb
import oe.recipeutils
- if args.recipename in workspace:
- logger.error("recipe %s is already in your workspace" % args.recipename)
- return -1
-
- reason = oe.recipeutils.validate_pn(args.recipename)
- if reason:
- logger.error(reason)
- return -1
+ if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri:
+ raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add')
+
+ # These are positional arguments, but because we're nice, allow
+ # specifying e.g. source tree without name, or fetch URI without name or
+ # source tree (if we can detect that that is what the user meant)
+ if scriptutils.is_src_url(args.recipename):
+ if not args.fetchuri:
+ if args.fetch:
+ raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
+ args.fetchuri = args.recipename
+ args.recipename = ''
+ elif scriptutils.is_src_url(args.srctree):
+ if not args.fetchuri:
+ if args.fetch:
+ raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
+ args.fetchuri = args.srctree
+ args.srctree = ''
+ elif args.recipename and not args.srctree:
+ if os.sep in args.recipename:
+ args.srctree = args.recipename
+ args.recipename = None
+ elif os.path.isdir(args.recipename):
+ logger.warn('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename)
+
+ if args.srctree and os.path.isfile(args.srctree):
+ args.fetchuri = 'file://' + os.path.abspath(args.srctree)
+ args.srctree = ''
+
+ if args.fetch:
+ if args.fetchuri:
+ raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
+ else:
+ logger.warn('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead')
+ args.fetchuri = args.fetch
- srctree = os.path.abspath(args.srctree)
- appendpath = os.path.join(config.workspace_path, 'appends')
- if not os.path.exists(appendpath):
- os.makedirs(appendpath)
+ if args.recipename:
+ if args.recipename in workspace:
+ raise DevtoolError("recipe %s is already in your workspace" %
+ args.recipename)
+ reason = oe.recipeutils.validate_pn(args.recipename)
+ if reason:
+ raise DevtoolError(reason)
+
+ if args.srctree:
+ srctree = os.path.abspath(args.srctree)
+ srctreeparent = None
+ tmpsrcdir = None
+ else:
+ srctree = None
+ srctreeparent = get_default_srctree(config)
+ bb.utils.mkdirhier(srctreeparent)
+ tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent)
+
+ if srctree and os.path.exists(srctree):
+ if args.fetchuri:
+ if not os.path.isdir(srctree):
+ raise DevtoolError("Cannot fetch into source tree path %s as "
+ "it exists and is not a directory" %
+ srctree)
+ elif os.listdir(srctree):
+ raise DevtoolError("Cannot fetch into source tree path %s as "
+ "it already exists and is non-empty" %
+ srctree)
+ elif not args.fetchuri:
+ if args.srctree:
+ raise DevtoolError("Specified source tree %s could not be found" %
+ args.srctree)
+ elif srctree:
+ raise DevtoolError("No source tree exists at default path %s - "
+ "either create and populate this directory, "
+ "or specify a path to a source tree, or a "
+ "URI to fetch source from" % srctree)
+ else:
+ raise DevtoolError("You must either specify a source tree "
+ "or a URI to fetch source from")
- recipedir = os.path.join(config.workspace_path, 'recipes', args.recipename)
- bb.utils.mkdirhier(recipedir)
if args.version:
if '_' in args.version or ' ' in args.version:
- logger.error('Invalid version string "%s"' % args.version)
- return -1
- bp = "%s_%s" % (args.recipename, args.version)
- else:
- bp = args.recipename
- recipefile = os.path.join(recipedir, "%s.bb" % bp)
- if sys.stdout.isatty():
+ raise DevtoolError('Invalid version string "%s"' % args.version)
+
+ if args.color == 'auto' and sys.stdout.isatty():
color = 'always'
else:
color = args.color
+ extracmdopts = ''
+ if args.fetchuri:
+ if args.fetchuri.startswith('npm://'):
+ ensure_npm(config, basepath, args.fixed_setup)
+
+ source = args.fetchuri
+ if srctree:
+ extracmdopts += ' -x %s' % srctree
+ else:
+ extracmdopts += ' -x %s' % tmpsrcdir
+ else:
+ source = srctree
+ if args.recipename:
+ extracmdopts += ' -N %s' % args.recipename
+ if args.version:
+ extracmdopts += ' -V %s' % args.version
+ if args.binary:
+ extracmdopts += ' -b'
+ if args.also_native:
+ extracmdopts += ' --also-native'
+ if args.src_subdir:
+ extracmdopts += ' --src-subdir "%s"' % args.src_subdir
+ if args.autorev:
+ extracmdopts += ' -a'
+ if args.fetch_dev:
+ extracmdopts += ' --fetch-dev'
+
+ tempdir = tempfile.mkdtemp(prefix='devtool')
try:
- stdout, stderr = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s "%s"' % (color, recipefile, srctree))
- logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
- except bb.process.ExecutionError as e:
- logger.error('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
- return 1
+ builtnpm = False
+ while True:
+ try:
+ stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True)
+ except bb.process.ExecutionError as e:
+ if e.exitcode == 14:
+ if builtnpm:
+ raise DevtoolError('Re-running recipetool still failed to find npm')
+ # FIXME this is a horrible hack that is unfortunately
+ # necessary due to the fact that we can't run bitbake from
+ # inside recipetool since recipetool keeps tinfoil active
+ # with references to it throughout the code, so we have
+ # to exit out and come back here to do it.
+ ensure_npm(config, basepath, args.fixed_setup, check_exists=False)
+ logger.info('Re-running recipe creation process after building nodejs')
+ builtnpm = True
+ continue
+ elif e.exitcode == 15:
+ raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line')
+ else:
+ raise DevtoolError('Command \'%s\' failed' % e.command)
+ break
+
+ recipes = glob.glob(os.path.join(tempdir, '*.bb'))
+ if recipes:
+ recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0]
+ if recipename in workspace:
+ raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename)
+ recipedir = os.path.join(config.workspace_path, 'recipes', recipename)
+ bb.utils.mkdirhier(recipedir)
+ recipefile = os.path.join(recipedir, os.path.basename(recipes[0]))
+ appendfile = recipe_to_append(recipefile, config)
+ if os.path.exists(appendfile):
+ # This shouldn't be possible, but just in case
+ raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace')
+ if os.path.exists(recipefile):
+ raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile)
+ if tmpsrcdir:
+ srctree = os.path.join(srctreeparent, recipename)
+ if os.path.exists(tmpsrcdir):
+ if os.path.exists(srctree):
+ if os.path.isdir(srctree):
+ try:
+ os.rmdir(srctree)
+ except OSError as e:
+ if e.errno == errno.ENOTEMPTY:
+ raise DevtoolError('Source tree path %s already exists and is not empty' % srctree)
+ else:
+ raise
+ else:
+ raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree)
+ logger.info('Using default source tree path %s' % srctree)
+ shutil.move(tmpsrcdir, srctree)
+ else:
+ raise DevtoolError('Couldn\'t find source tree created by recipetool')
+ bb.utils.mkdirhier(recipedir)
+ shutil.move(recipes[0], recipefile)
+ # Move any additional files created by recipetool
+ for fn in os.listdir(tempdir):
+ shutil.move(os.path.join(tempdir, fn), recipedir)
+ else:
+ raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout))
+ attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile))
+ if os.path.exists(attic_recipe):
+ logger.warn('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe)
+ finally:
+ if tmpsrcdir and os.path.exists(tmpsrcdir):
+ shutil.rmtree(tmpsrcdir)
+ shutil.rmtree(tempdir)
- _add_md5(config, args.recipename, recipefile)
+ for fn in os.listdir(recipedir):
+ _add_md5(config, recipename, os.path.join(recipedir, fn))
- initial_rev = None
- if os.path.exists(os.path.join(srctree, '.git')):
- (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
- initial_rev = stdout.rstrip()
+ tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
+ try:
+ try:
+ rd = tinfoil.parse_recipe_file(recipefile, False)
+ except Exception as e:
+ logger.error(str(e))
+ rd = None
+ if not rd:
+ # Parsing failed. We just created this recipe and we shouldn't
+ # leave it in the workdir or it'll prevent bitbake from starting
+ movefn = '%s.parsefailed' % recipefile
+ logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn)
+ shutil.move(recipefile, movefn)
+ return 1
+
+ if args.fetchuri and not args.no_git:
+ setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data)
+
+ initial_rev = None
+ if os.path.exists(os.path.join(srctree, '.git')):
+ (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
+ initial_rev = stdout.rstrip()
- appendfile = os.path.join(appendpath, '%s.bbappend' % bp)
- with open(appendfile, 'w') as f:
- f.write('inherit externalsrc\n')
- f.write('EXTERNALSRC = "%s"\n' % srctree)
- if args.same_dir:
- f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree)
- if initial_rev:
- f.write('\n# initial_rev: %s\n' % initial_rev)
+ if args.src_subdir:
+ srctree = os.path.join(srctree, args.src_subdir)
- _add_md5(config, args.recipename, appendfile)
+ bb.utils.mkdirhier(os.path.dirname(appendfile))
+ with open(appendfile, 'w') as f:
+ f.write('inherit externalsrc\n')
+ f.write('EXTERNALSRC = "%s"\n' % srctree)
+
+ b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd)
+ if b_is_s:
+ f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree)
+ if initial_rev:
+ f.write('\n# initial_rev: %s\n' % initial_rev)
+
+ if args.binary:
+ f.write('do_install_append() {\n')
+ f.write(' rm -rf ${D}/.git\n')
+ f.write(' rm -f ${D}/singletask.lock\n')
+ f.write('}\n')
+
+ if bb.data.inherits_class('npm', rd):
+ f.write('do_install_append() {\n')
+ f.write(' # Remove files added to source dir by devtool/externalsrc\n')
+ f.write(' rm -f ${NPM_INSTALLDIR}/singletask.lock\n')
+ f.write(' rm -rf ${NPM_INSTALLDIR}/.git\n')
+ f.write(' rm -rf ${NPM_INSTALLDIR}/oe-local-files\n')
+ f.write(' for symlink in ${EXTERNALSRC_SYMLINKS} ; do\n')
+ f.write(' rm -f ${NPM_INSTALLDIR}/${symlink%%:*}\n')
+ f.write(' done\n')
+ f.write('}\n')
+
+ _add_md5(config, recipename, appendfile)
+
+ logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
+
+ finally:
+ tinfoil.shutdown()
return 0
def _check_compatible_recipe(pn, d):
+ """Check if the recipe is supported by devtool"""
if pn == 'perf':
- logger.error("The perf recipe does not actually check out source and thus cannot be supported by this tool")
- return False
+ raise DevtoolError("The perf recipe does not actually check out "
+ "source and thus cannot be supported by this tool",
+ 4)
- if pn in ['gcc-source', 'kernel-devsrc', 'package-index']:
- logger.error("The %s recipe is not supported by this tool" % pn)
- return False
+ if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'):
+ raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4)
if bb.data.inherits_class('image', d):
- logger.error("The %s recipe is an image, and therefore is not supported by this tool" % pn)
- return False
+ raise DevtoolError("The %s recipe is an image, and therefore is not "
+ "supported by this tool" % pn, 4)
if bb.data.inherits_class('populate_sdk', d):
- logger.error("The %s recipe is an SDK, and therefore is not supported by this tool" % pn)
- return False
+ raise DevtoolError("The %s recipe is an SDK, and therefore is not "
+ "supported by this tool" % pn, 4)
if bb.data.inherits_class('packagegroup', d):
- logger.error("The %s recipe is a packagegroup, and therefore is not supported by this tool" % pn)
- return False
+ raise DevtoolError("The %s recipe is a packagegroup, and therefore is "
+ "not supported by this tool" % pn, 4)
if bb.data.inherits_class('meta', d):
- logger.error("The %s recipe is a meta-recipe, and therefore is not supported by this tool" % pn)
- return False
-
- if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC', True):
- logger.error("externalsrc is currently enabled for the %s recipe. This prevents the normal do_patch task from working. You will need to disable this first." % pn)
- return False
-
- return True
-
+ raise DevtoolError("The %s recipe is a meta-recipe, and therefore is "
+ "not supported by this tool" % pn, 4)
+
+ if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'):
+ # Not an incompatibility error per se, so we don't pass the error code
+ raise DevtoolError("externalsrc is currently enabled for the %s "
+ "recipe. This prevents the normal do_patch task "
+ "from working. You will need to disable this "
+ "first." % pn)
+
+def _move_file(src, dst):
+ """Move a file. Creates all the directory components of destination path."""
+ dst_d = os.path.dirname(dst)
+ if dst_d:
+ bb.utils.mkdirhier(dst_d)
+ shutil.move(src, dst)
+
+def _copy_file(src, dst):
+ """Copy a file. Creates all the directory components of destination path."""
+ dst_d = os.path.dirname(dst)
+ if dst_d:
+ bb.utils.mkdirhier(dst_d)
+ shutil.copy(src, dst)
+
+def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
+ """List contents of a git treeish"""
+ import bb
+ cmd = ['git', 'ls-tree', '-z', treeish]
+ if recursive:
+ cmd.append('-r')
+ out, _ = bb.process.run(cmd, cwd=repodir)
+ ret = {}
+ if out:
+ for line in out.split('\0'):
+ if line:
+ split = line.split(None, 4)
+ ret[split[3]] = split[0:3]
+ return ret
+
+def _git_exclude_path(srctree, path):
+ """Return pathspec (list of paths) that excludes certain path"""
+ # NOTE: "Filtering out" files/paths in this way is not entirely reliable -
+ # we don't catch files that are deleted, for example. A more reliable way
+ # to implement this would be to use "negative pathspecs" which were
+ # introduced in Git v1.9.0. Revisit this when/if the required Git version
+ # becomes greater than that.
+ path = os.path.normpath(path)
+ recurse = True if len(path.split(os.path.sep)) > 1 else False
+ git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys())
+ if path in git_files:
+ git_files.remove(path)
+ return git_files
+ else:
+ return ['.']
-def _get_recipe_file(cooker, pn):
- import oe.recipeutils
- recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
- if not recipefile:
- skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
- if skipreasons:
- logger.error('\n'.join(skipreasons))
- else:
- logger.error("Unable to find any recipe file matching %s" % pn)
- return recipefile
+def _ls_tree(directory):
+ """Recursive listing of files in a directory"""
+ ret = []
+ for root, dirs, files in os.walk(directory):
+ ret.extend([os.path.relpath(os.path.join(root, fname), directory) for
+ fname in files])
+ return ret
-def _parse_recipe(config, tinfoil, pn, appends):
- """Parse recipe of a package"""
- import oe.recipeutils
- recipefile = _get_recipe_file(tinfoil.cooker, pn)
- if not recipefile:
- # Error already logged
- return None
- if appends:
- append_files = tinfoil.cooker.collection.get_file_appends(recipefile)
- # Filter out appends from the workspace
- append_files = [path for path in append_files if
- not path.startswith(config.workspace_path)]
- return oe.recipeutils.parse_recipe(recipefile, append_files,
- tinfoil.config_data)
def extract(args, config, basepath, workspace):
+ """Entry point for the devtool 'extract' subcommand"""
import bb
- tinfoil = setup_tinfoil()
+ tinfoil = _prep_extract_operation(config, basepath, args.recipename)
+ if not tinfoil:
+ # Error already shown
+ return 1
+ try:
+ rd = parse_recipe(config, tinfoil, args.recipename, True)
+ if not rd:
+ return 1
- rd = _parse_recipe(config, tinfoil, args.recipename, True)
- if not rd:
- return -1
+ srctree = os.path.abspath(args.srctree)
+ initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil)
+ logger.info('Source tree extracted to %s' % srctree)
- srctree = os.path.abspath(args.srctree)
- initial_rev = _extract_source(srctree, args.keep_temp, args.branch, rd)
- if initial_rev:
- return 0
- else:
- return -1
+ if initial_rev:
+ return 0
+ else:
+ return 1
+ finally:
+ tinfoil.shutdown()
+
+def sync(args, config, basepath, workspace):
+ """Entry point for the devtool 'sync' subcommand"""
+ import bb
+ tinfoil = _prep_extract_operation(config, basepath, args.recipename)
+ if not tinfoil:
+ # Error already shown
+ return 1
+ try:
+ rd = parse_recipe(config, tinfoil, args.recipename, True)
+ if not rd:
+ return 1
-def _extract_source(srctree, keep_temp, devbranch, d):
- import bb.event
+ srctree = os.path.abspath(args.srctree)
+ initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd, tinfoil)
+ logger.info('Source tree %s synchronized' % srctree)
- def eventfilter(name, handler, event, d):
- if name == 'base_eventhandler':
- return True
+ if initial_rev:
+ return 0
else:
- return False
+ return 1
+ finally:
+ tinfoil.shutdown()
- if hasattr(bb.event, 'set_eventfilter'):
- bb.event.set_eventfilter(eventfilter)
- pn = d.getVar('PN', True)
+def _prep_extract_operation(config, basepath, recipename, tinfoil=None):
+ """HACK: Ugly workaround for making sure that requirements are met when
+ trying to extract a package. Returns the tinfoil instance to be used."""
+ if not tinfoil:
+ tinfoil = setup_tinfoil(basepath=basepath)
- if not _check_compatible_recipe(pn, d):
+ rd = parse_recipe(config, tinfoil, recipename, True)
+ if not rd:
return None
- if os.path.exists(srctree):
- if not os.path.isdir(srctree):
- logger.error("output path %s exists and is not a directory" % srctree)
- return None
- elif os.listdir(srctree):
- logger.error("output path %s already exists and is non-empty" % srctree)
- return None
+ if bb.data.inherits_class('kernel-yocto', rd):
+ tinfoil.shutdown()
+ try:
+ stdout, _ = exec_build_env_command(config.init_path, basepath,
+ 'bitbake kern-tools-native')
+ tinfoil = setup_tinfoil(basepath=basepath)
+ except bb.process.ExecutionError as err:
+ raise DevtoolError("Failed to build kern-tools-native:\n%s" %
+ err.stdout)
+ return tinfoil
+
+
+def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil):
+ """Extract sources of a recipe"""
+ import oe.recipeutils
+
+ pn = d.getVar('PN')
- # Prepare for shutil.move later on
- bb.utils.mkdirhier(srctree)
- os.rmdir(srctree)
+ _check_compatible_recipe(pn, d)
+
+ if sync:
+ if not os.path.exists(srctree):
+ raise DevtoolError("output path %s does not exist" % srctree)
+ else:
+ if os.path.exists(srctree):
+ if not os.path.isdir(srctree):
+ raise DevtoolError("output path %s exists and is not a directory" %
+ srctree)
+ elif os.listdir(srctree):
+ raise DevtoolError("output path %s already exists and is "
+ "non-empty" % srctree)
+
+ if 'noexec' in (d.getVarFlags('do_unpack', False) or []):
+ raise DevtoolError("The %s recipe has do_unpack disabled, unable to "
+ "extract source" % pn, 4)
+
+ if not sync:
+ # Prepare for shutil.move later on
+ bb.utils.mkdirhier(srctree)
+ os.rmdir(srctree)
initial_rev = None
- tempdir = tempfile.mkdtemp(prefix='devtool')
+ # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary
+ # directory so that:
+ # (a) we pick up all files that get unpacked to the WORKDIR, and
+ # (b) we don't disturb the existing build
+ # However, with recipe-specific sysroots the sysroots for the recipe
+ # will be prepared under WORKDIR, and if we used the system temporary
+ # directory (i.e. usually /tmp) as used by mkdtemp by default, then
+ # our attempts to hardlink files into the recipe-specific sysroots
+ # will fail on systems where /tmp is a different filesystem, and it
+ # would have to fall back to copying the files which is a waste of
+ # time. Put the temp directory under the WORKDIR to prevent that from
+ # being a problem.
+ tempbasedir = d.getVar('WORKDIR')
+ bb.utils.mkdirhier(tempbasedir)
+ tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir)
try:
+ tinfoil.logger.setLevel(logging.WARNING)
+
crd = d.createCopy()
# Make a subdir so we guard against WORKDIR==S
workdir = os.path.join(tempdir, 'workdir')
crd.setVar('WORKDIR', workdir)
- crd.setVar('T', os.path.join(tempdir, 'temp'))
- if not crd.getVar('S', True).startswith(workdir):
+ if not crd.getVar('S').startswith(workdir):
# Usually a shared workdir recipe (kernel, gcc)
# Try to set a reasonable default
if bb.data.inherits_class('kernel', d):
crd.setVar('S', '${WORKDIR}/source')
else:
- crd.setVar('S', '${WORKDIR}/${BP}')
+ crd.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S')))
if bb.data.inherits_class('kernel', d):
# We don't want to move the source to STAGING_KERNEL_DIR here
crd.setVar('STAGING_KERNEL_DIR', '${S}')
- # FIXME: This is very awkward. Unfortunately it's not currently easy to properly
- # execute tasks outside of bitbake itself, until then this has to suffice if we
- # are to handle e.g. linux-yocto's extra tasks
- executed = []
- def exec_task_func(func, report):
- if not func in executed:
- deps = crd.getVarFlag(func, 'deps')
- if deps:
- for taskdepfunc in deps:
- exec_task_func(taskdepfunc, True)
- if report:
- logger.info('Executing %s...' % func)
- fn = d.getVar('FILE', True)
- localdata = bb.build._task_data(fn, func, crd)
- bb.build.exec_func(func, localdata)
- executed.append(func)
-
- logger.info('Fetching %s...' % pn)
- exec_task_func('do_fetch', False)
- logger.info('Unpacking...')
- exec_task_func('do_unpack', False)
- srcsubdir = crd.getVar('S', True)
- patchsubdir = srcsubdir
- if srcsubdir != workdir and os.path.dirname(srcsubdir) != workdir:
+ is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d)
+ if not is_kernel_yocto:
+ crd.setVar('PATCHTOOL', 'git')
+ crd.setVar('PATCH_COMMIT_FUNCTIONS', '1')
+
+ # Apply our changes to the datastore to the server's datastore
+ for key in crd.localkeys():
+ tinfoil.config_data.setVar('%s_pn-%s' % (key, pn), crd.getVar(key, False))
+
+ tinfoil.config_data.setVar('STAMPS_DIR', os.path.join(tempdir, 'stamps'))
+ tinfoil.config_data.setVar('T', os.path.join(tempdir, 'temp'))
+ tinfoil.config_data.setVar('BUILDCFG_FUNCS', '')
+ tinfoil.config_data.setVar('BUILDCFG_HEADER', '')
+ tinfoil.config_data.setVar('BB_HASH_IGNORE_MISMATCH', '1')
+
+ tinfoil.set_event_mask(['bb.event.BuildStarted',
+ 'bb.event.BuildCompleted',
+ 'logging.LogRecord',
+ 'bb.command.CommandCompleted',
+ 'bb.command.CommandFailed',
+ 'bb.build.TaskStarted',
+ 'bb.build.TaskSucceeded',
+ 'bb.build.TaskFailed',
+ 'bb.build.TaskFailedSilent'])
+
+ def runtask(target, task):
+ if tinfoil.build_file(target, task):
+ while True:
+ event = tinfoil.wait_event(0.25)
+ if event:
+ if isinstance(event, bb.command.CommandCompleted):
+ break
+ elif isinstance(event, bb.command.CommandFailed):
+ raise DevtoolError('Task do_%s failed: %s' % (task, event.error))
+ elif isinstance(event, bb.build.TaskFailed):
+ raise DevtoolError('Task do_%s failed' % task)
+ elif isinstance(event, bb.build.TaskStarted):
+ logger.info('Executing %s...' % event._task)
+ elif isinstance(event, logging.LogRecord):
+ if event.levelno <= logging.INFO:
+ continue
+ logger.handle(event)
+
+ # we need virtual:native:/path/to/recipe if it's a BBCLASSEXTEND
+ fn = tinfoil.get_recipe_file(pn)
+ runtask(fn, 'unpack')
+
+ if bb.data.inherits_class('kernel-yocto', d):
+ # Extra step for kernel to populate the source directory
+ runtask(fn, 'kernel_checkout')
+
+ srcsubdir = crd.getVar('S')
+
+ # Move local source files into separate subdir
+ recipe_patches = [os.path.basename(patch) for patch in
+ oe.recipeutils.get_recipe_patches(crd)]
+ local_files = oe.recipeutils.get_recipe_local_files(crd)
+
+ # Ignore local files with subdir={BP}
+ srcabspath = os.path.abspath(srcsubdir)
+ local_files = [fname for fname in local_files if
+ os.path.exists(os.path.join(workdir, fname)) and
+ (srcabspath == workdir or not
+ os.path.join(workdir, fname).startswith(srcabspath +
+ os.sep))]
+ if local_files:
+ for fname in local_files:
+ _move_file(os.path.join(workdir, fname),
+ os.path.join(tempdir, 'oe-local-files', fname))
+ with open(os.path.join(tempdir, 'oe-local-files', '.gitignore'),
+ 'w') as f:
+ f.write('# Ignore local files, by default. Remove this file '
+ 'if you want to commit the directory to Git\n*\n')
+
+ if srcsubdir == workdir:
+ # Find non-patch non-local sources that were "unpacked" to srctree
+ # directory
+ src_files = [fname for fname in _ls_tree(workdir) if
+ os.path.basename(fname) not in recipe_patches]
+ # Force separate S so that patch files can be left out from srctree
+ srcsubdir = tempfile.mkdtemp(dir=workdir)
+ tinfoil.config_data.setVar('S_task-patch', srcsubdir)
+ # Move source files to S
+ for path in src_files:
+ _move_file(os.path.join(workdir, path),
+ os.path.join(srcsubdir, path))
+ elif os.path.dirname(srcsubdir) != workdir:
# Handle if S is set to a subdirectory of the source
srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
scriptutils.git_convert_standalone_clone(srcsubdir)
- patchdir = os.path.join(patchsubdir, 'patches')
- haspatches = False
- if os.path.exists(patchdir):
- if os.listdir(patchdir):
- haspatches = True
- else:
- os.rmdir(patchdir)
+ # Make sure that srcsubdir exists
+ bb.utils.mkdirhier(srcsubdir)
+ if not os.path.exists(srcsubdir) or not os.listdir(srcsubdir):
+ logger.warning("no source unpacked to S, either the %s recipe "
+ "doesn't use any source or the correct source "
+ "directory could not be determined" % pn)
- if bb.data.inherits_class('kernel-yocto', d):
- (stdout, _) = bb.process.run('git --git-dir="%s" rev-parse HEAD' % crd.expand('${WORKDIR}/git'), cwd=srcsubdir)
- initial_rev = stdout.rstrip()
- else:
- if not os.listdir(srcsubdir):
- logger.error("no source unpacked to S, perhaps the %s recipe doesn't use any source?" % pn)
- return None
+ setup_git_repo(srcsubdir, crd.getVar('PV'), devbranch, d=d)
- if not os.path.exists(os.path.join(srcsubdir, '.git')):
- bb.process.run('git init', cwd=srcsubdir)
- bb.process.run('git add .', cwd=srcsubdir)
- bb.process.run('git commit -q -m "Initial commit from upstream at version %s"' % crd.getVar('PV', True), cwd=srcsubdir)
+ (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
+ initial_rev = stdout.rstrip()
- (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
- initial_rev = stdout.rstrip()
+ logger.info('Patching...')
+ runtask(fn, 'patch')
- bb.process.run('git checkout -b %s' % devbranch, cwd=srcsubdir)
- bb.process.run('git tag -f devtool-base', cwd=srcsubdir)
+ bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
- crd.setVar('PATCHTOOL', 'git')
+ kconfig = None
+ if bb.data.inherits_class('kernel-yocto', d):
+ # Store generate and store kernel config
+ logger.info('Generating kernel config')
+ runtask(fn, 'configure')
+ kconfig = os.path.join(crd.getVar('B'), '.config')
- logger.info('Patching...')
- exec_task_func('do_patch', False)
- bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
+ tempdir_localdir = os.path.join(tempdir, 'oe-local-files')
+ srctree_localdir = os.path.join(srctree, 'oe-local-files')
- if os.path.exists(patchdir):
- shutil.rmtree(patchdir)
- if haspatches:
- bb.process.run('git checkout patches', cwd=srcsubdir)
+ if sync:
+ bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree)
+
+ # Move oe-local-files directory to srctree
+ # As the oe-local-files is not part of the constructed git tree,
+ # remove them directly during the synchrounizating might surprise
+ # the users. Instead, we move it to oe-local-files.bak and remind
+ # user in the log message.
+ if os.path.exists(srctree_localdir + '.bak'):
+ shutil.rmtree(srctree_localdir, srctree_localdir + '.bak')
+
+ if os.path.exists(srctree_localdir):
+ logger.info('Backing up current local file directory %s' % srctree_localdir)
+ shutil.move(srctree_localdir, srctree_localdir + '.bak')
+
+ if os.path.exists(tempdir_localdir):
+ logger.info('Syncing local source files to srctree...')
+ shutil.copytree(tempdir_localdir, srctree_localdir)
+ else:
+ # Move oe-local-files directory to srctree
+ if os.path.exists(tempdir_localdir):
+ logger.info('Adding local source files to srctree...')
+ shutil.move(tempdir_localdir, srcsubdir)
+
+ shutil.move(srcsubdir, srctree)
+
+ if os.path.abspath(d.getVar('S')) == os.path.abspath(d.getVar('WORKDIR')):
+ # If recipe extracts to ${WORKDIR}, symlink the files into the srctree
+ # (otherwise the recipe won't build as expected)
+ local_files_dir = os.path.join(srctree, 'oe-local-files')
+ addfiles = []
+ for root, _, files in os.walk(local_files_dir):
+ relpth = os.path.relpath(root, local_files_dir)
+ if relpth != '.':
+ bb.utils.mkdirhier(os.path.join(srctree, relpth))
+ for fn in files:
+ if fn == '.gitignore':
+ continue
+ destpth = os.path.join(srctree, relpth, fn)
+ if os.path.exists(destpth):
+ os.unlink(destpth)
+ os.symlink('oe-local-files/%s' % fn, destpth)
+ addfiles.append(os.path.join(relpth, fn))
+ if addfiles:
+ bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree)
+ useroptions = []
+ oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d)
+ bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree)
+
+ if kconfig:
+ logger.info('Copying kernel config to srctree')
+ shutil.copy2(kconfig, srctree)
- shutil.move(srcsubdir, srctree)
- logger.info('Source tree extracted to %s' % srctree)
finally:
if keep_temp:
logger.info('Preserving temporary directory %s' % tempdir)
@@ -293,23 +694,42 @@ def _extract_source(srctree, keep_temp, devbranch, d):
return initial_rev
def _add_md5(config, recipename, filename):
+ """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace"""
import bb.utils
- md5 = bb.utils.md5_file(filename)
- with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a') as f:
- f.write('%s|%s|%s\n' % (recipename, os.path.relpath(filename, config.workspace_path), md5))
+
+ def addfile(fn):
+ md5 = bb.utils.md5_file(fn)
+ with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a') as f:
+ f.write('%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5))
+
+ if os.path.isdir(filename):
+ for root, _, files in os.walk(filename):
+ for f in files:
+ addfile(os.path.join(root, f))
+ else:
+ addfile(filename)
def _check_preserve(config, recipename):
+ """Check if a file was manually changed and needs to be saved in 'attic'
+ directory"""
import bb.utils
origfile = os.path.join(config.workspace_path, '.devtool_md5')
newfile = os.path.join(config.workspace_path, '.devtool_md5_new')
- preservepath = os.path.join(config.workspace_path, 'attic')
+ preservepath = os.path.join(config.workspace_path, 'attic', recipename)
with open(origfile, 'r') as f:
with open(newfile, 'w') as tf:
for line in f.readlines():
splitline = line.rstrip().split('|')
if splitline[0] == recipename:
removefile = os.path.join(config.workspace_path, splitline[1])
- md5 = bb.utils.md5_file(removefile)
+ try:
+ md5 = bb.utils.md5_file(removefile)
+ except IOError as err:
+ if err.errno == 2:
+ # File no longer exists, skip it
+ continue
+ else:
+ raise
if splitline[2] != md5:
bb.utils.mkdirhier(preservepath)
preservefile = os.path.basename(removefile)
@@ -321,414 +741,1178 @@ def _check_preserve(config, recipename):
tf.write(line)
os.rename(newfile, origfile)
- return False
-
-
def modify(args, config, basepath, workspace):
+ """Entry point for the devtool 'modify' subcommand"""
import bb
import oe.recipeutils
if args.recipename in workspace:
- logger.error("recipe %s is already in your workspace" % args.recipename)
- return -1
+ raise DevtoolError("recipe %s is already in your workspace" %
+ args.recipename)
- if not args.extract:
- if not os.path.isdir(args.srctree):
- logger.error("directory %s does not exist or not a directory (specify -x to extract source from recipe)" % args.srctree)
- return -1
-
- tinfoil = setup_tinfoil()
+ tinfoil = setup_tinfoil(basepath=basepath)
+ try:
+ rd = parse_recipe(config, tinfoil, args.recipename, True)
+ if not rd:
+ return 1
+
+ pn = rd.getVar('PN')
+ if pn != args.recipename:
+ logger.info('Mapping %s to %s' % (args.recipename, pn))
+ if pn in workspace:
+ raise DevtoolError("recipe %s is already in your workspace" %
+ pn)
+
+ if args.srctree:
+ srctree = os.path.abspath(args.srctree)
+ else:
+ srctree = get_default_srctree(config, pn)
+
+ if args.no_extract and not os.path.isdir(srctree):
+ raise DevtoolError("--no-extract specified and source path %s does "
+ "not exist or is not a directory" %
+ srctree)
+ if not args.no_extract:
+ tinfoil = _prep_extract_operation(config, basepath, pn, tinfoil)
+ if not tinfoil:
+ # Error already shown
+ return 1
+ # We need to re-parse because tinfoil may have been re-initialised
+ rd = parse_recipe(config, tinfoil, args.recipename, True)
- rd = _parse_recipe(config, tinfoil, args.recipename, True)
- if not rd:
- return -1
- recipefile = rd.getVar('FILE', True)
+ recipefile = rd.getVar('FILE')
+ appendfile = recipe_to_append(recipefile, config, args.wildcard)
+ if os.path.exists(appendfile):
+ raise DevtoolError("Another variant of recipe %s is already in your "
+ "workspace (only one variant of a recipe can "
+ "currently be worked on at once)"
+ % pn)
- if not _check_compatible_recipe(args.recipename, rd):
- return -1
+ _check_compatible_recipe(pn, rd)
- initial_rev = None
- commits = []
- srctree = os.path.abspath(args.srctree)
- if args.extract:
- initial_rev = _extract_source(args.srctree, False, args.branch, rd)
- if not initial_rev:
- return -1
- # Get list of commits since this revision
- (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=args.srctree)
- commits = stdout.split()
- else:
- if os.path.exists(os.path.join(args.srctree, '.git')):
- # Check if it's a tree previously extracted by us
- try:
- (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=args.srctree)
- except bb.process.ExecutionError:
- stdout = ''
- for line in stdout.splitlines():
- if line.startswith('*'):
- (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=args.srctree)
- initial_rev = stdout.rstrip()
+ initial_rev = None
+ commits = []
+ if not args.no_extract:
+ initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil)
if not initial_rev:
- # Otherwise, just grab the head revision
- (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=args.srctree)
- initial_rev = stdout.rstrip()
-
- # Check that recipe isn't using a shared workdir
- s = rd.getVar('S', True)
- workdir = rd.getVar('WORKDIR', True)
- if s.startswith(workdir):
- # Handle if S is set to a subdirectory of the source
- if s != workdir and os.path.dirname(s) != workdir:
- srcsubdir = os.sep.join(os.path.relpath(s, workdir).split(os.sep)[1:])
+ return 1
+ logger.info('Source tree extracted to %s' % srctree)
+ # Get list of commits since this revision
+ (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree)
+ commits = stdout.split()
+ else:
+ if os.path.exists(os.path.join(srctree, '.git')):
+ # Check if it's a tree previously extracted by us
+ try:
+ (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree)
+ except bb.process.ExecutionError:
+ stdout = ''
+ for line in stdout.splitlines():
+ if line.startswith('*'):
+ (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree)
+ initial_rev = stdout.rstrip()
+ if not initial_rev:
+ # Otherwise, just grab the head revision
+ (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
+ initial_rev = stdout.rstrip()
+
+ # Check that recipe isn't using a shared workdir
+ s = os.path.abspath(rd.getVar('S'))
+ workdir = os.path.abspath(rd.getVar('WORKDIR'))
+ if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir:
+ # Handle if S is set to a subdirectory of the source
+ srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1]
srctree = os.path.join(srctree, srcsubdir)
- appendpath = os.path.join(config.workspace_path, 'appends')
- if not os.path.exists(appendpath):
- os.makedirs(appendpath)
-
- appendname = os.path.splitext(os.path.basename(recipefile))[0]
- if args.wildcard:
- appendname = re.sub(r'_.*', '_%', appendname)
- appendfile = os.path.join(appendpath, appendname + '.bbappend')
- with open(appendfile, 'w') as f:
- f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
- f.write('inherit externalsrc\n')
- f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n')
- f.write('EXTERNALSRC_pn-%s = "%s"\n' % (args.recipename, srctree))
-
- b_is_s = True
- if args.no_same_dir:
- logger.info('using separate build directory since --no-same-dir specified')
- b_is_s = False
- elif args.same_dir:
- logger.info('using source tree as build directory since --same-dir specified')
- elif bb.data.inherits_class('autotools-brokensep', rd):
- logger.info('using source tree as build directory since original recipe inherits autotools-brokensep')
- elif rd.getVar('B', True) == s:
- logger.info('using source tree as build directory since that is the default for this recipe')
- else:
- b_is_s = False
- if b_is_s:
- f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (args.recipename, srctree))
+ bb.utils.mkdirhier(os.path.dirname(appendfile))
+ with open(appendfile, 'w') as f:
+ f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n')
+ # Local files can be modified/tracked in separate subdir under srctree
+ # Mostly useful for packages with S != WORKDIR
+ f.write('FILESPATH_prepend := "%s:"\n' %
+ os.path.join(srctree, 'oe-local-files'))
+
+ f.write('\ninherit externalsrc\n')
+ f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n')
+ f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree))
+
+ b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd)
+ if b_is_s:
+ f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree))
+
+ if bb.data.inherits_class('kernel', rd):
+ f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout '
+ 'do_fetch do_unpack do_patch do_kernel_configme do_kernel_configcheck"\n')
+ f.write('\ndo_configure_append() {\n'
+ ' cp ${B}/.config ${S}/.config.baseline\n'
+ ' ln -sfT ${B}/.config ${S}/.config.new\n'
+ '}\n')
+ if initial_rev:
+ f.write('\n# initial_rev: %s\n' % initial_rev)
+ for commit in commits:
+ f.write('# commit: %s\n' % commit)
- if initial_rev:
- f.write('\n# initial_rev: %s\n' % initial_rev)
- for commit in commits:
- f.write('# commit: %s\n' % commit)
+ _add_md5(config, pn, appendfile)
- _add_md5(config, args.recipename, appendfile)
+ logger.info('Recipe %s now set up to build from %s' % (pn, srctree))
- logger.info('Recipe %s now set up to build from %s' % (args.recipename, srctree))
+ finally:
+ tinfoil.shutdown()
return 0
-def update_recipe(args, config, basepath, workspace):
- if not args.recipename in workspace:
- logger.error("no recipe named %s in your workspace" % args.recipename)
- return -1
-
- tinfoil = setup_tinfoil()
+def rename(args, config, basepath, workspace):
+ """Entry point for the devtool 'rename' subcommand"""
import bb
- from oe.patch import GitApplyTree
import oe.recipeutils
- rd = _parse_recipe(config, tinfoil, args.recipename, True)
- if not rd:
- return -1
- recipefile = rd.getVar('FILE', True)
+ check_workspace_recipe(workspace, args.recipename)
- # Get initial revision from bbappend
- append = os.path.join(config.workspace_path, 'appends', '%s.bbappend' % os.path.splitext(os.path.basename(recipefile))[0])
- if not os.path.exists(append):
- logger.error('unable to find workspace bbappend for recipe %s' % args.recipename)
- return -1
+ if not (args.newname or args.version):
+ raise DevtoolError('You must specify a new name, a version with -V/--version, or both')
- orig_src_uri = rd.getVar('SRC_URI', False) or ''
- if args.mode == 'auto':
- if 'git://' in orig_src_uri:
- mode = 'srcrev'
+ recipefile = workspace[args.recipename]['recipefile']
+ if not recipefile:
+ raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)')
+
+ if args.newname and args.newname != args.recipename:
+ reason = oe.recipeutils.validate_pn(args.newname)
+ if reason:
+ raise DevtoolError(reason)
+ newname = args.newname
+ else:
+ newname = args.recipename
+
+ append = workspace[args.recipename]['bbappend']
+ appendfn = os.path.splitext(os.path.basename(append))[0]
+ splitfn = appendfn.split('_')
+ if len(splitfn) > 1:
+ origfnver = appendfn.split('_')[1]
+ else:
+ origfnver = ''
+
+ recipefilemd5 = None
+ tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
+ try:
+ rd = parse_recipe(config, tinfoil, args.recipename, True)
+ if not rd:
+ return 1
+
+ bp = rd.getVar('BP')
+ bpn = rd.getVar('BPN')
+ if newname != args.recipename:
+ localdata = rd.createCopy()
+ localdata.setVar('PN', newname)
+ newbpn = localdata.getVar('BPN')
else:
- mode = 'patch'
+ newbpn = bpn
+ s = rd.getVar('S', False)
+ src_uri = rd.getVar('SRC_URI', False)
+ pv = rd.getVar('PV')
+
+ # Correct variable values that refer to the upstream source - these
+ # values must stay the same, so if the name/version are changing then
+ # we need to fix them up
+ new_s = s
+ new_src_uri = src_uri
+ if newbpn != bpn:
+ # ${PN} here is technically almost always incorrect, but people do use it
+ new_s = new_s.replace('${BPN}', bpn)
+ new_s = new_s.replace('${PN}', bpn)
+ new_s = new_s.replace('${BP}', '%s-${PV}' % bpn)
+ new_src_uri = new_src_uri.replace('${BPN}', bpn)
+ new_src_uri = new_src_uri.replace('${PN}', bpn)
+ new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn)
+ if args.version and origfnver == pv:
+ new_s = new_s.replace('${PV}', pv)
+ new_s = new_s.replace('${BP}', '${BPN}-%s' % pv)
+ new_src_uri = new_src_uri.replace('${PV}', pv)
+ new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv)
+ patchfields = {}
+ if new_s != s:
+ patchfields['S'] = new_s
+ if new_src_uri != src_uri:
+ patchfields['SRC_URI'] = new_src_uri
+ if patchfields:
+ recipefilemd5 = bb.utils.md5_file(recipefile)
+ oe.recipeutils.patch_recipe(rd, recipefile, patchfields)
+ newrecipefilemd5 = bb.utils.md5_file(recipefile)
+ finally:
+ tinfoil.shutdown()
+
+ if args.version:
+ newver = args.version
else:
- mode = args.mode
-
- def remove_patches(srcuri, patchlist):
- # Remove any patches that we don't need
- updated = False
- for patch in patchlist:
- patchfile = os.path.basename(patch)
- for i in xrange(len(srcuri)):
- if srcuri[i].startswith('file://') and os.path.basename(srcuri[i]).split(';')[0] == patchfile:
- logger.info('Removing patch %s' % patchfile)
- srcuri.pop(i)
- # FIXME "git rm" here would be nice if the file in question is tracked
- # FIXME there's a chance that this file is referred to by another recipe, in which case deleting wouldn't be the right thing to do
- if patch.startswith(os.path.dirname(recipefile)):
- os.remove(patch)
- updated = True
- break
- return updated
-
- srctree = workspace[args.recipename]
+ newver = origfnver
+
+ if newver:
+ newappend = '%s_%s.bbappend' % (newname, newver)
+ newfile = '%s_%s.bb' % (newname, newver)
+ else:
+ newappend = '%s.bbappend' % newname
+ newfile = '%s.bb' % newname
+
+ oldrecipedir = os.path.dirname(recipefile)
+ newrecipedir = os.path.join(config.workspace_path, 'recipes', newname)
+ if oldrecipedir != newrecipedir:
+ bb.utils.mkdirhier(newrecipedir)
+
+ newappend = os.path.join(os.path.dirname(append), newappend)
+ newfile = os.path.join(newrecipedir, newfile)
+
+ # Rename bbappend
+ logger.info('Renaming %s to %s' % (append, newappend))
+ os.rename(append, newappend)
+ # Rename recipe file
+ logger.info('Renaming %s to %s' % (recipefile, newfile))
+ os.rename(recipefile, newfile)
+
+ # Rename source tree if it's the default path
+ appendmd5 = None
+ if not args.no_srctree:
+ srctree = workspace[args.recipename]['srctree']
+ if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename):
+ newsrctree = os.path.join(config.workspace_path, 'sources', newname)
+ logger.info('Renaming %s to %s' % (srctree, newsrctree))
+ shutil.move(srctree, newsrctree)
+ # Correct any references (basically EXTERNALSRC*) in the .bbappend
+ appendmd5 = bb.utils.md5_file(newappend)
+ appendlines = []
+ with open(newappend, 'r') as f:
+ for line in f:
+ appendlines.append(line)
+ with open(newappend, 'w') as f:
+ for line in appendlines:
+ if srctree in line:
+ line = line.replace(srctree, newsrctree)
+ f.write(line)
+ newappendmd5 = bb.utils.md5_file(newappend)
+
+ bpndir = None
+ newbpndir = None
+ if newbpn != bpn:
+ bpndir = os.path.join(oldrecipedir, bpn)
+ if os.path.exists(bpndir):
+ newbpndir = os.path.join(newrecipedir, newbpn)
+ logger.info('Renaming %s to %s' % (bpndir, newbpndir))
+ shutil.move(bpndir, newbpndir)
+
+ bpdir = None
+ newbpdir = None
+ if newver != origfnver or newbpn != bpn:
+ bpdir = os.path.join(oldrecipedir, bp)
+ if os.path.exists(bpdir):
+ newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver))
+ logger.info('Renaming %s to %s' % (bpdir, newbpdir))
+ shutil.move(bpdir, newbpdir)
+
+ if oldrecipedir != newrecipedir:
+ # Move any stray files and delete the old recipe directory
+ for entry in os.listdir(oldrecipedir):
+ oldpath = os.path.join(oldrecipedir, entry)
+ newpath = os.path.join(newrecipedir, entry)
+ logger.info('Renaming %s to %s' % (oldpath, newpath))
+ shutil.move(oldpath, newpath)
+ os.rmdir(oldrecipedir)
+
+ # Now take care of entries in .devtool_md5
+ md5entries = []
+ with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f:
+ for line in f:
+ md5entries.append(line)
+
+ if bpndir and newbpndir:
+ relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/'
+ else:
+ relbpndir = None
+ if bpdir and newbpdir:
+ relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/'
+ else:
+ relbpdir = None
+
+ with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f:
+ for entry in md5entries:
+ splitentry = entry.rstrip().split('|')
+ if len(splitentry) > 2:
+ if splitentry[0] == args.recipename:
+ splitentry[0] = newname
+ if splitentry[1] == os.path.relpath(append, config.workspace_path):
+ splitentry[1] = os.path.relpath(newappend, config.workspace_path)
+ if appendmd5 and splitentry[2] == appendmd5:
+ splitentry[2] = newappendmd5
+ elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path):
+ splitentry[1] = os.path.relpath(newfile, config.workspace_path)
+ if recipefilemd5 and splitentry[2] == recipefilemd5:
+ splitentry[2] = newrecipefilemd5
+ elif relbpndir and splitentry[1].startswith(relbpndir):
+ splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path)
+ elif relbpdir and splitentry[1].startswith(relbpdir):
+ splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path)
+ entry = '|'.join(splitentry) + '\n'
+ f.write(entry)
+ return 0
+
+
+def _get_patchset_revs(srctree, recipe_path, initial_rev=None):
+ """Get initial and update rev of a recipe. These are the start point of the
+ whole patchset and start point for the patches to be re-generated/updated.
+ """
+ import bb
+
+ # Parse initial rev from recipe if not specified
+ commits = []
+ with open(recipe_path, 'r') as f:
+ for line in f:
+ if line.startswith('# initial_rev:'):
+ if not initial_rev:
+ initial_rev = line.split(':')[-1].strip()
+ elif line.startswith('# commit:'):
+ commits.append(line.split(':')[-1].strip())
+
+ update_rev = initial_rev
+ changed_revs = None
+ if initial_rev:
+ # Find first actually changed revision
+ stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' %
+ initial_rev, cwd=srctree)
+ newcommits = stdout.split()
+ for i in range(min(len(commits), len(newcommits))):
+ if newcommits[i] == commits[i]:
+ update_rev = commits[i]
+
+ try:
+ stdout, _ = bb.process.run('git cherry devtool-patched',
+ cwd=srctree)
+ except bb.process.ExecutionError as err:
+ stdout = None
+
+ if stdout is not None:
+ changed_revs = []
+ for line in stdout.splitlines():
+ if line.startswith('+ '):
+ rev = line.split()[1]
+ if rev in newcommits:
+ changed_revs.append(rev)
+
+ return initial_rev, update_rev, changed_revs
+
+def _remove_file_entries(srcuri, filelist):
+ """Remove file:// entries from SRC_URI"""
+ remaining = filelist[:]
+ entries = []
+ for fname in filelist:
+ basename = os.path.basename(fname)
+ for i in range(len(srcuri)):
+ if (srcuri[i].startswith('file://') and
+ os.path.basename(srcuri[i].split(';')[0]) == basename):
+ entries.append(srcuri[i])
+ remaining.remove(fname)
+ srcuri.pop(i)
+ break
+ return entries, remaining
+
+def _replace_srcuri_entry(srcuri, filename, newentry):
+ """Replace entry corresponding to specified file with a new entry"""
+ basename = os.path.basename(filename)
+ for i in range(len(srcuri)):
+ if os.path.basename(srcuri[i].split(';')[0]) == basename:
+ srcuri.pop(i)
+ srcuri.insert(i, newentry)
+ break
+
+def _remove_source_files(append, files, destpath):
+ """Unlink existing patch files"""
+ for path in files:
+ if append:
+ if not destpath:
+ raise Exception('destpath should be set here')
+ path = os.path.join(destpath, os.path.basename(path))
+
+ if os.path.exists(path):
+ logger.info('Removing file %s' % path)
+ # FIXME "git rm" here would be nice if the file in question is
+ # tracked
+ # FIXME there's a chance that this file is referred to by
+ # another recipe, in which case deleting wouldn't be the
+ # right thing to do
+ os.remove(path)
+ # Remove directory if empty
+ try:
+ os.rmdir(os.path.dirname(path))
+ except OSError as ose:
+ if ose.errno != errno.ENOTEMPTY:
+ raise
+
+
+def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
+ """Export patches from srctree to given location.
+ Returns three-tuple of dicts:
+ 1. updated - patches that already exist in SRCURI
+ 2. added - new patches that don't exist in SRCURI
+ 3 removed - patches that exist in SRCURI but not in exported patches
+ In each dict the key is the 'basepath' of the URI and value is the
+ absolute path to the existing file in recipe space (if any).
+ """
+ import oe.recipeutils
+ from oe.patch import GitApplyTree
+ updated = OrderedDict()
+ added = OrderedDict()
+ seqpatch_re = re.compile('^([0-9]{4}-)?(.+)')
+
+ existing_patches = dict((os.path.basename(path), path) for path in
+ oe.recipeutils.get_recipe_patches(rd))
+
+ # Generate patches from Git, exclude local files directory
+ patch_pathspec = _git_exclude_path(srctree, 'oe-local-files')
+ GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec)
+
+ new_patches = sorted(os.listdir(destdir))
+ for new_patch in new_patches:
+ # Strip numbering from patch names. If it's a git sequence named patch,
+ # the numbers might not match up since we are starting from a different
+ # revision This does assume that people are using unique shortlog
+ # values, but they ought to be anyway...
+ new_basename = seqpatch_re.match(new_patch).group(2)
+ match_name = None
+ for old_patch in existing_patches:
+ old_basename = seqpatch_re.match(old_patch).group(2)
+ old_basename_splitext = os.path.splitext(old_basename)
+ if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename:
+ old_patch_noext = os.path.splitext(old_patch)[0]
+ match_name = old_patch_noext
+ break
+ elif new_basename == old_basename:
+ match_name = old_patch
+ break
+ if match_name:
+ # Rename patch files
+ if new_patch != match_name:
+ os.rename(os.path.join(destdir, new_patch),
+ os.path.join(destdir, match_name))
+ # Need to pop it off the list now before checking changed_revs
+ oldpath = existing_patches.pop(old_patch)
+ if changed_revs is not None:
+ # Avoid updating patches that have not actually changed
+ with open(os.path.join(destdir, match_name), 'r') as f:
+ firstlineitems = f.readline().split()
+ # Looking for "From <hash>" line
+ if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40:
+ if not firstlineitems[1] in changed_revs:
+ continue
+ # Recompress if necessary
+ if oldpath.endswith(('.gz', '.Z')):
+ bb.process.run(['gzip', match_name], cwd=destdir)
+ if oldpath.endswith('.gz'):
+ match_name += '.gz'
+ else:
+ match_name += '.Z'
+ elif oldpath.endswith('.bz2'):
+ bb.process.run(['bzip2', match_name], cwd=destdir)
+ match_name += '.bz2'
+ updated[match_name] = oldpath
+ else:
+ added[new_patch] = None
+ return (updated, added, existing_patches)
+
+
+def _create_kconfig_diff(srctree, rd, outfile):
+ """Create a kconfig fragment"""
+ # Only update config fragment if both config files exist
+ orig_config = os.path.join(srctree, '.config.baseline')
+ new_config = os.path.join(srctree, '.config.new')
+ if os.path.exists(orig_config) and os.path.exists(new_config):
+ cmd = ['diff', '--new-line-format=%L', '--old-line-format=',
+ '--unchanged-line-format=', orig_config, new_config]
+ pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = pipe.communicate()
+ if pipe.returncode == 1:
+ logger.info("Updating config fragment %s" % outfile)
+ with open(outfile, 'wb') as fobj:
+ fobj.write(stdout)
+ elif pipe.returncode == 0:
+ logger.info("Would remove config fragment %s" % outfile)
+ if os.path.exists(outfile):
+ # Remove fragment file in case of empty diff
+ logger.info("Removing config fragment %s" % outfile)
+ os.unlink(outfile)
+ else:
+ raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr)
+ return True
+ return False
+
+
+def _export_local_files(srctree, rd, destdir):
+ """Copy local files from srctree to given location.
+ Returns three-tuple of dicts:
+ 1. updated - files that already exist in SRCURI
+ 2. added - new files files that don't exist in SRCURI
+ 3 removed - files that exist in SRCURI but not in exported files
+ In each dict the key is the 'basepath' of the URI and value is the
+ absolute path to the existing file in recipe space (if any).
+ """
+ import oe.recipeutils
+
+ # Find out local files (SRC_URI files that exist in the "recipe space").
+ # Local files that reside in srctree are not included in patch generation.
+ # Instead they are directly copied over the original source files (in
+ # recipe space).
+ existing_files = oe.recipeutils.get_recipe_local_files(rd)
+ new_set = None
+ updated = OrderedDict()
+ added = OrderedDict()
+ removed = OrderedDict()
+ local_files_dir = os.path.join(srctree, 'oe-local-files')
+ git_files = _git_ls_tree(srctree)
+ if 'oe-local-files' in git_files:
+ # If tracked by Git, take the files from srctree HEAD. First get
+ # the tree object of the directory
+ tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool')
+ tree = git_files['oe-local-files'][2]
+ bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree,
+ env=dict(os.environ, GIT_WORK_TREE=destdir,
+ GIT_INDEX_FILE=tmp_index))
+ new_set = list(_git_ls_tree(srctree, tree, True).keys())
+ elif os.path.isdir(local_files_dir):
+ # If not tracked by Git, just copy from working copy
+ new_set = _ls_tree(os.path.join(srctree, 'oe-local-files'))
+ bb.process.run(['cp', '-ax',
+ os.path.join(srctree, 'oe-local-files', '.'), destdir])
+ else:
+ new_set = []
+
+ # Special handling for kernel config
+ if bb.data.inherits_class('kernel-yocto', rd):
+ fragment_fn = 'devtool-fragment.cfg'
+ fragment_path = os.path.join(destdir, fragment_fn)
+ if _create_kconfig_diff(srctree, rd, fragment_path):
+ if os.path.exists(fragment_path):
+ if fragment_fn not in new_set:
+ new_set.append(fragment_fn)
+ # Copy fragment to local-files
+ if os.path.isdir(local_files_dir):
+ shutil.copy2(fragment_path, local_files_dir)
+ else:
+ if fragment_fn in new_set:
+ new_set.remove(fragment_fn)
+ # Remove fragment from local-files
+ if os.path.exists(os.path.join(local_files_dir, fragment_fn)):
+ os.unlink(os.path.join(local_files_dir, fragment_fn))
+
+ if new_set is not None:
+ for fname in new_set:
+ if fname in existing_files:
+ origpath = existing_files.pop(fname)
+ workpath = os.path.join(local_files_dir, fname)
+ if not filecmp.cmp(origpath, workpath):
+ updated[fname] = origpath
+ elif fname != '.gitignore':
+ added[fname] = None
+
+ workdir = rd.getVar('WORKDIR')
+ s = rd.getVar('S')
+ if not s.endswith(os.sep):
+ s += os.sep
+
+ if workdir != s:
+ # Handle files where subdir= was specified
+ for fname in list(existing_files.keys()):
+ # FIXME handle both subdir starting with BP and not?
+ fworkpath = os.path.join(workdir, fname)
+ if fworkpath.startswith(s):
+ fpath = os.path.join(srctree, os.path.relpath(fworkpath, s))
+ if os.path.exists(fpath):
+ origpath = existing_files.pop(fname)
+ if not filecmp.cmp(origpath, fpath):
+ updated[fpath] = origpath
+
+ removed = existing_files
+ return (updated, added, removed)
+
+
+def _determine_files_dir(rd):
+ """Determine the appropriate files directory for a recipe"""
+ recipedir = rd.getVar('FILE_DIRNAME')
+ for entry in rd.getVar('FILESPATH').split(':'):
+ relpth = os.path.relpath(entry, recipedir)
+ if not os.sep in relpth:
+ # One (or zero) levels below only, so we don't put anything in machine-specific directories
+ if os.path.isdir(entry):
+ return entry
+ return os.path.join(recipedir, rd.getVar('BPN'))
+
+
+def _update_recipe_srcrev(srctree, rd, appendlayerdir, wildcard_version, no_remove):
+ """Implement the 'srcrev' mode of update-recipe"""
+ import bb
+ import oe.recipeutils
+
+ recipefile = rd.getVar('FILE')
+ logger.info('Updating SRCREV in recipe %s' % os.path.basename(recipefile))
# Get HEAD revision
try:
- (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
+ stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree)
except bb.process.ExecutionError as err:
- print('Failed to get HEAD revision in %s: %s' % (srctree, err))
- return 1
+ raise DevtoolError('Failed to get HEAD revision in %s: %s' %
+ (srctree, err))
srcrev = stdout.strip()
if len(srcrev) != 40:
- logger.error('Invalid hash returned by git: %s' % stdout)
- return 1
+ raise DevtoolError('Invalid hash returned by git: %s' % stdout)
- if mode == 'srcrev':
- logger.info('Updating SRCREV in recipe %s' % os.path.basename(recipefile))
- patchfields = {}
- patchfields['SRCREV'] = srcrev
- if not args.no_remove:
+ destpath = None
+ remove_files = []
+ patchfields = {}
+ patchfields['SRCREV'] = srcrev
+ orig_src_uri = rd.getVar('SRC_URI', False) or ''
+ srcuri = orig_src_uri.split()
+ tempdir = tempfile.mkdtemp(prefix='devtool')
+ update_srcuri = False
+ try:
+ local_files_dir = tempfile.mkdtemp(dir=tempdir)
+ upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir)
+ if not no_remove:
# Find list of existing patches in recipe file
- existing_patches = oe.recipeutils.get_recipe_patches(rd)
-
+ patches_dir = tempfile.mkdtemp(dir=tempdir)
old_srcrev = (rd.getVar('SRCREV', False) or '')
- tempdir = tempfile.mkdtemp(prefix='devtool')
- removepatches = []
- try:
- GitApplyTree.extractPatches(srctree, old_srcrev, tempdir)
- newpatches = os.listdir(tempdir)
- for patch in existing_patches:
- patchfile = os.path.basename(patch)
- if patchfile in newpatches:
- removepatches.append(patch)
- finally:
- shutil.rmtree(tempdir)
- if removepatches:
- srcuri = (rd.getVar('SRC_URI', False) or '').split()
- if remove_patches(srcuri, removepatches):
- patchfields['SRC_URI'] = ' '.join(srcuri)
-
- oe.recipeutils.patch_recipe(tinfoil.config_data, recipefile, patchfields)
-
- if not 'git://' in orig_src_uri:
- logger.info('You will need to update SRC_URI within the recipe to point to a git repository where you have pushed your changes')
-
- elif mode == 'patch':
- commits = []
- update_rev = None
- if args.initial_rev:
- initial_rev = args.initial_rev
+ upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev,
+ patches_dir)
+
+ # Remove deleted local files and "overlapping" patches
+ remove_files = list(del_f.values()) + list(upd_p.values())
+ if remove_files:
+ removedentries = _remove_file_entries(srcuri, remove_files)[0]
+ update_srcuri = True
+
+ if appendlayerdir:
+ files = dict((os.path.join(local_files_dir, key), val) for
+ key, val in list(upd_f.items()) + list(new_f.items()))
+ removevalues = {}
+ if update_srcuri:
+ removevalues = {'SRC_URI': removedentries}
+ patchfields['SRC_URI'] = '\\\n '.join(srcuri)
+ _, destpath = oe.recipeutils.bbappend_recipe(
+ rd, appendlayerdir, files, wildcardver=wildcard_version,
+ extralines=patchfields, removevalues=removevalues)
else:
- initial_rev = None
- with open(append, 'r') as f:
- for line in f:
- if line.startswith('# initial_rev:'):
- initial_rev = line.split(':')[-1].strip()
- elif line.startswith('# commit:'):
- commits.append(line.split(':')[-1].strip())
+ files_dir = _determine_files_dir(rd)
+ for basepath, path in upd_f.items():
+ logger.info('Updating file %s' % basepath)
+ if os.path.isabs(basepath):
+ # Original file (probably with subdir pointing inside source tree)
+ # so we do not want to move it, just copy
+ _copy_file(basepath, path)
+ else:
+ _move_file(os.path.join(local_files_dir, basepath), path)
+ update_srcuri= True
+ for basepath, path in new_f.items():
+ logger.info('Adding new file %s' % basepath)
+ _move_file(os.path.join(local_files_dir, basepath),
+ os.path.join(files_dir, basepath))
+ srcuri.append('file://%s' % basepath)
+ update_srcuri = True
+ if update_srcuri:
+ patchfields['SRC_URI'] = ' '.join(srcuri)
+ oe.recipeutils.patch_recipe(rd, recipefile, patchfields)
+ finally:
+ shutil.rmtree(tempdir)
+ if not 'git://' in orig_src_uri:
+ logger.info('You will need to update SRC_URI within the recipe to '
+ 'point to a git repository where you have pushed your '
+ 'changes')
- if initial_rev:
- # Find first actually changed revision
- (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree)
- newcommits = stdout.split()
- for i in xrange(min(len(commits), len(newcommits))):
- if newcommits[i] == commits[i]:
- update_rev = commits[i]
+ _remove_source_files(appendlayerdir, remove_files, destpath)
+ return True
- if not initial_rev:
- logger.error('Unable to find initial revision - please specify it with --initial-rev')
- return -1
+def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, initial_rev):
+ """Implement the 'patch' mode of update-recipe"""
+ import bb
+ import oe.recipeutils
+
+ recipefile = rd.getVar('FILE')
+ append = workspace[recipename]['bbappend']
+ if not os.path.exists(append):
+ raise DevtoolError('unable to find workspace bbappend for recipe %s' %
+ recipename)
- if not update_rev:
- update_rev = initial_rev
+ initial_rev, update_rev, changed_revs = _get_patchset_revs(srctree, append, initial_rev)
+ if not initial_rev:
+ raise DevtoolError('Unable to find initial revision - please specify '
+ 'it with --initial-rev')
- # Find list of existing patches in recipe file
- existing_patches = oe.recipeutils.get_recipe_patches(rd)
+ dl_dir = rd.getVar('DL_DIR')
+ if not dl_dir.endswith('/'):
+ dl_dir += '/'
- removepatches = []
- if not args.no_remove:
+ tempdir = tempfile.mkdtemp(prefix='devtool')
+ try:
+ local_files_dir = tempfile.mkdtemp(dir=tempdir)
+ upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir)
+
+ remove_files = []
+ if not no_remove:
# Get all patches from source tree and check if any should be removed
- tempdir = tempfile.mkdtemp(prefix='devtool')
- try:
- GitApplyTree.extractPatches(srctree, initial_rev, tempdir)
- newpatches = os.listdir(tempdir)
- for patch in existing_patches:
- patchfile = os.path.basename(patch)
- if patchfile not in newpatches:
- removepatches.append(patch)
- finally:
- shutil.rmtree(tempdir)
+ all_patches_dir = tempfile.mkdtemp(dir=tempdir)
+ upd_p, new_p, del_p = _export_patches(srctree, rd, initial_rev,
+ all_patches_dir)
+ # Remove deleted local files and patches
+ remove_files = list(del_f.values()) + list(del_p.values())
# Get updated patches from source tree
- tempdir = tempfile.mkdtemp(prefix='devtool')
- try:
- GitApplyTree.extractPatches(srctree, update_rev, tempdir)
-
- # Match up and replace existing patches with corresponding new patches
- updatepatches = False
- updaterecipe = False
- newpatches = os.listdir(tempdir)
- for patch in existing_patches:
- patchfile = os.path.basename(patch)
- if patchfile in newpatches:
- logger.info('Updating patch %s' % patchfile)
- shutil.move(os.path.join(tempdir, patchfile), patch)
- newpatches.remove(patchfile)
- updatepatches = True
- srcuri = (rd.getVar('SRC_URI', False) or '').split()
- if newpatches:
- # Add any patches left over
- patchdir = os.path.join(os.path.dirname(recipefile), rd.getVar('BPN', True))
- bb.utils.mkdirhier(patchdir)
- for patchfile in newpatches:
- logger.info('Adding new patch %s' % patchfile)
- shutil.move(os.path.join(tempdir, patchfile), os.path.join(patchdir, patchfile))
- srcuri.append('file://%s' % patchfile)
- updaterecipe = True
- if removepatches:
- if remove_patches(srcuri, removepatches):
+ patches_dir = tempfile.mkdtemp(dir=tempdir)
+ upd_p, new_p, del_p = _export_patches(srctree, rd, update_rev,
+ patches_dir, changed_revs)
+ updatefiles = False
+ updaterecipe = False
+ destpath = None
+ srcuri = (rd.getVar('SRC_URI', False) or '').split()
+ if appendlayerdir:
+ files = dict((os.path.join(local_files_dir, key), val) for
+ key, val in list(upd_f.items()) + list(new_f.items()))
+ files.update(dict((os.path.join(patches_dir, key), val) for
+ key, val in list(upd_p.items()) + list(new_p.items())))
+ if files or remove_files:
+ removevalues = None
+ if remove_files:
+ removedentries, remaining = _remove_file_entries(
+ srcuri, remove_files)
+ if removedentries or remaining:
+ remaining = ['file://' + os.path.basename(item) for
+ item in remaining]
+ removevalues = {'SRC_URI': removedentries + remaining}
+ _, destpath = oe.recipeutils.bbappend_recipe(
+ rd, appendlayerdir, files,
+ wildcardver=wildcard_version,
+ removevalues=removevalues)
+ else:
+ logger.info('No patches or local source files needed updating')
+ else:
+ # Update existing files
+ files_dir = _determine_files_dir(rd)
+ for basepath, path in upd_f.items():
+ logger.info('Updating file %s' % basepath)
+ if os.path.isabs(basepath):
+ # Original file (probably with subdir pointing inside source tree)
+ # so we do not want to move it, just copy
+ _copy_file(basepath, path)
+ else:
+ _move_file(os.path.join(local_files_dir, basepath), path)
+ updatefiles = True
+ for basepath, path in upd_p.items():
+ patchfn = os.path.join(patches_dir, basepath)
+ if os.path.dirname(path) + '/' == dl_dir:
+ # This is a a downloaded patch file - we now need to
+ # replace the entry in SRC_URI with our local version
+ logger.info('Replacing remote patch %s with updated local version' % basepath)
+ path = os.path.join(files_dir, basepath)
+ _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath)
updaterecipe = True
+ else:
+ logger.info('Updating patch %s' % basepath)
+ logger.debug('Moving new patch %s to %s' % (patchfn, path))
+ _move_file(patchfn, path)
+ updatefiles = True
+ # Add any new files
+ for basepath, path in new_f.items():
+ logger.info('Adding new file %s' % basepath)
+ _move_file(os.path.join(local_files_dir, basepath),
+ os.path.join(files_dir, basepath))
+ srcuri.append('file://%s' % basepath)
+ updaterecipe = True
+ for basepath, path in new_p.items():
+ logger.info('Adding new patch %s' % basepath)
+ _move_file(os.path.join(patches_dir, basepath),
+ os.path.join(files_dir, basepath))
+ srcuri.append('file://%s' % basepath)
+ updaterecipe = True
+ # Update recipe, if needed
+ if _remove_file_entries(srcuri, remove_files)[0]:
+ updaterecipe = True
if updaterecipe:
logger.info('Updating recipe %s' % os.path.basename(recipefile))
- oe.recipeutils.patch_recipe(tinfoil.config_data,
- recipefile, {'SRC_URI': ' '.join(srcuri)})
- elif not updatepatches:
+ oe.recipeutils.patch_recipe(rd, recipefile,
+ {'SRC_URI': ' '.join(srcuri)})
+ elif not updatefiles:
# Neither patches nor recipe were updated
- logger.info('No patches need updating')
- finally:
- shutil.rmtree(tempdir)
+ logger.info('No patches or files need updating')
+ return False
+ finally:
+ shutil.rmtree(tempdir)
+ _remove_source_files(appendlayerdir, remove_files, destpath)
+ return True
+
+def _guess_recipe_update_mode(srctree, rdata):
+ """Guess the recipe update mode to use"""
+ src_uri = (rdata.getVar('SRC_URI', False) or '').split()
+ git_uris = [uri for uri in src_uri if uri.startswith('git://')]
+ if not git_uris:
+ return 'patch'
+ # Just use the first URI for now
+ uri = git_uris[0]
+ # Check remote branch
+ params = bb.fetch.decodeurl(uri)[5]
+ upstr_branch = params['branch'] if 'branch' in params else 'master'
+ # Check if current branch HEAD is found in upstream branch
+ stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree)
+ head_rev = stdout.rstrip()
+ stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev,
+ cwd=srctree)
+ remote_brs = [branch.strip() for branch in stdout.splitlines()]
+ if 'origin/' + upstr_branch in remote_brs:
+ return 'srcrev'
+
+ return 'patch'
+
+def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev):
+ srctree = workspace[recipename]['srctree']
+ if mode == 'auto':
+ mode = _guess_recipe_update_mode(srctree, rd)
+
+ if mode == 'srcrev':
+ updated = _update_recipe_srcrev(srctree, rd, appendlayerdir, wildcard_version, no_remove)
+ elif mode == 'patch':
+ updated = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, initial_rev)
else:
- logger.error('update_recipe: invalid mode %s' % mode)
- return 1
+ raise DevtoolError('update_recipe: invalid mode %s' % mode)
+ return updated
+
+def update_recipe(args, config, basepath, workspace):
+ """Entry point for the devtool 'update-recipe' subcommand"""
+ check_workspace_recipe(workspace, args.recipename)
+
+ if args.append:
+ if not os.path.exists(args.append):
+ raise DevtoolError('bbappend destination layer directory "%s" '
+ 'does not exist' % args.append)
+ if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')):
+ raise DevtoolError('conf/layer.conf not found in bbappend '
+ 'destination layer "%s"' % args.append)
+
+ tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
+ try:
+
+ rd = parse_recipe(config, tinfoil, args.recipename, True)
+ if not rd:
+ return 1
+
+ updated = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev)
+
+ if updated:
+ rf = rd.getVar('FILE')
+ if rf.startswith(config.workspace_path):
+ logger.warn('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf)
+ finally:
+ tinfoil.shutdown()
return 0
def status(args, config, basepath, workspace):
+ """Entry point for the devtool 'status' subcommand"""
if workspace:
- for recipe, value in workspace.iteritems():
- print("%s: %s" % (recipe, value))
+ for recipe, value in workspace.items():
+ recipefile = value['recipefile']
+ if recipefile:
+ recipestr = ' (%s)' % recipefile
+ else:
+ recipestr = ''
+ print("%s: %s%s" % (recipe, value['srctree'], recipestr))
else:
logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one')
return 0
-def reset(args, config, basepath, workspace):
- import bb
- if args.recipename:
- if args.all:
- logger.error("Recipe cannot be specified if -a/--all is used")
- return -1
- elif not args.recipename in workspace:
- logger.error("no recipe named %s in your workspace" % args.recipename)
- return -1
- elif not args.all:
- logger.error("Recipe must be specified, or specify -a/--all to reset all recipes")
- return -1
+def _reset(recipes, no_clean, config, basepath, workspace):
+ """Reset one or more recipes"""
- if args.all:
- recipes = workspace
- else:
- recipes = [args.recipename]
+ if recipes and not no_clean:
+ if len(recipes) == 1:
+ logger.info('Cleaning sysroot for recipe %s...' % recipes[0])
+ else:
+ logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes))
+ # If the recipe file itself was created in the workspace, and
+ # it uses BBCLASSEXTEND, then we need to also clean the other
+ # variants
+ targets = []
+ for recipe in recipes:
+ targets.append(recipe)
+ recipefile = workspace[recipe]['recipefile']
+ if recipefile and os.path.exists(recipefile):
+ targets.extend(get_bbclassextend_targets(recipefile, recipe))
+ try:
+ exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets))
+ except bb.process.ExecutionError as e:
+ raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you '
+ 'wish, you may specify -n/--no-clean to '
+ 'skip running this command when resetting' %
+ (e.command, e.stdout))
for pn in recipes:
- if not args.no_clean:
- logger.info('Cleaning sysroot for recipe %s...' % pn)
- try:
- exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % pn)
- except bb.process.ExecutionError as e:
- logger.error('Command \'%s\' failed, output:\n%s\nIf you wish, you may specify -n/--no-clean to skip running this command when resetting' % (e.command, e.stdout))
- return 1
-
_check_preserve(config, pn)
- preservepath = os.path.join(config.workspace_path, 'attic', pn)
+ preservepath = os.path.join(config.workspace_path, 'attic', pn, pn)
def preservedir(origdir):
if os.path.exists(origdir):
- for fn in os.listdir(origdir):
- logger.warn('Preserving %s in %s' % (fn, preservepath))
- bb.utils.mkdirhier(preservepath)
- shutil.move(os.path.join(origdir, fn), os.path.join(preservepath, fn))
+ for root, dirs, files in os.walk(origdir):
+ for fn in files:
+ logger.warn('Preserving %s in %s' % (fn, preservepath))
+ _move_file(os.path.join(origdir, fn),
+ os.path.join(preservepath, fn))
+ for dn in dirs:
+ preservedir(os.path.join(root, dn))
os.rmdir(origdir)
preservedir(os.path.join(config.workspace_path, 'recipes', pn))
# We don't automatically create this dir next to appends, but the user can
preservedir(os.path.join(config.workspace_path, 'appends', pn))
+ srctree = workspace[pn]['srctree']
+ if os.path.isdir(srctree):
+ if os.listdir(srctree):
+ # We don't want to risk wiping out any work in progress
+ logger.info('Leaving source tree %s as-is; if you no '
+ 'longer need it then please delete it manually'
+ % srctree)
+ else:
+ # This is unlikely, but if it's empty we can just remove it
+ os.rmdir(srctree)
+
+
+def reset(args, config, basepath, workspace):
+ """Entry point for the devtool 'reset' subcommand"""
+ import bb
+ if args.recipename:
+ if args.all:
+ raise DevtoolError("Recipe cannot be specified if -a/--all is used")
+ else:
+ for recipe in args.recipename:
+ check_workspace_recipe(workspace, recipe, checksrc=False)
+ elif not args.all:
+ raise DevtoolError("Recipe must be specified, or specify -a/--all to "
+ "reset all recipes")
+ if args.all:
+ recipes = list(workspace.keys())
+ else:
+ recipes = args.recipename
+
+ _reset(recipes, args.no_clean, config, basepath, workspace)
+
return 0
-def build(args, config, basepath, workspace):
+def _get_layer(layername, d):
+ """Determine the base layer path for the specified layer name/path"""
+ layerdirs = d.getVar('BBLAYERS').split()
+ layers = {os.path.basename(p): p for p in layerdirs}
+ # Provide some shortcuts
+ if layername.lower() in ['oe-core', 'openembedded-core']:
+ layerdir = layers.get('meta', None)
+ else:
+ layerdir = layers.get(layername, None)
+ if layerdir:
+ layerdir = os.path.abspath(layerdir)
+ return layerdir or layername
+
+def finish(args, config, basepath, workspace):
+ """Entry point for the devtool 'finish' subcommand"""
import bb
- if not args.recipename in workspace:
- logger.error("no recipe named %s in your workspace" % args.recipename)
- return -1
- build_task = config.get('Build', 'build_task', 'populate_sysroot')
+ import oe.recipeutils
+
+ check_workspace_recipe(workspace, args.recipename)
+
+ no_clean = False
+ tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
try:
- exec_build_env_command(config.init_path, basepath, 'bitbake -c %s %s' % (build_task, args.recipename), watch=True)
- except bb.process.ExecutionError as e:
- # We've already seen the output since watch=True, so just ensure we return something to the user
- return e.exitcode
+ rd = parse_recipe(config, tinfoil, args.recipename, True)
+ if not rd:
+ return 1
+
+ destlayerdir = _get_layer(args.destination, tinfoil.config_data)
+ origlayerdir = oe.recipeutils.find_layerdir(rd.getVar('FILE'))
+
+ if not os.path.isdir(destlayerdir):
+ raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination)
+
+ if os.path.abspath(destlayerdir) == config.workspace_path:
+ raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination)
+
+ # If it's an upgrade, grab the original path
+ origpath = None
+ origfilelist = None
+ append = workspace[args.recipename]['bbappend']
+ with open(append, 'r') as f:
+ for line in f:
+ if line.startswith('# original_path:'):
+ origpath = line.split(':')[1].strip()
+ elif line.startswith('# original_files:'):
+ origfilelist = line.split(':')[1].split()
+
+ if origlayerdir == config.workspace_path:
+ # Recipe file itself is in workspace, update it there first
+ appendlayerdir = None
+ origrelpath = None
+ if origpath:
+ origlayerpath = oe.recipeutils.find_layerdir(origpath)
+ if origlayerpath:
+ origrelpath = os.path.relpath(origpath, origlayerpath)
+ destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath)
+ if not destpath:
+ raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir))
+ # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases)
+ layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
+ if not os.path.abspath(destlayerdir) in layerdirs:
+ bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename)
+
+ elif destlayerdir == origlayerdir:
+ # Same layer, update the original recipe
+ appendlayerdir = None
+ destpath = None
+ else:
+ # Create/update a bbappend in the specified layer
+ appendlayerdir = destlayerdir
+ destpath = None
+
+ # Remove any old files in the case of an upgrade
+ if origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == oe.recipeutils.find_layerdir(destlayerdir):
+ for fn in origfilelist:
+ fnp = os.path.join(origpath, fn)
+ try:
+ os.remove(fnp)
+ except FileNotFoundError:
+ pass
+
+ # Actually update the recipe / bbappend
+ _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, initial_rev=args.initial_rev)
+
+ if origlayerdir == config.workspace_path and destpath:
+ # Recipe file itself is in the workspace - need to move it and any
+ # associated files to the specified layer
+ no_clean = True
+ logger.info('Moving recipe file to %s' % destpath)
+ recipedir = os.path.dirname(rd.getVar('FILE'))
+ for root, _, files in os.walk(recipedir):
+ for fn in files:
+ srcpath = os.path.join(root, fn)
+ relpth = os.path.relpath(os.path.dirname(srcpath), recipedir)
+ destdir = os.path.abspath(os.path.join(destpath, relpth))
+ bb.utils.mkdirhier(destdir)
+ shutil.move(srcpath, os.path.join(destdir, fn))
+
+ finally:
+ tinfoil.shutdown()
+
+ # Everything else has succeeded, we can now reset
+ _reset([args.recipename], no_clean=no_clean, config=config, basepath=basepath, workspace=workspace)
return 0
+def get_default_srctree(config, recipename=''):
+ """Get the default srctree path"""
+ srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path)
+ if recipename:
+ return os.path.join(srctreeparent, 'sources', recipename)
+ else:
+ return os.path.join(srctreeparent, 'sources')
+
def register_commands(subparsers, context):
+ """Register devtool subcommands from this plugin"""
+
+ defsrctree = get_default_srctree(context.config)
parser_add = subparsers.add_parser('add', help='Add a new recipe',
- description='Adds a new recipe',
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser_add.add_argument('recipename', help='Name for new recipe to add')
- parser_add.add_argument('srctree', help='Path to external source tree')
- parser_add.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
+ description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.',
+ group='starting', order=100)
+ parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.')
+ parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
+ parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree')
+ group = parser_add.add_mutually_exclusive_group()
+ group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
+ group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
+ parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI')
+ parser_add.add_argument('--fetch-dev', help='For npm, also fetch devDependencies', action="store_true")
parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
- parser_add.set_defaults(func=add)
+ parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true")
+ parser_add.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
+ parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true')
+ parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true')
+ parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
+ parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup)
parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe',
- description='Enables modifying the source for an existing recipe',
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser_modify.add_argument('recipename', help='Name for recipe to edit')
- parser_modify.add_argument('srctree', help='Path to external source tree')
+ description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.',
+ group='starting', order=90)
+ parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)')
+ parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend')
- parser_modify.add_argument('--extract', '-x', action="store_true", help='Extract source as well')
+ group = parser_modify.add_mutually_exclusive_group()
+ group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)')
+ group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist')
group = parser_modify.add_mutually_exclusive_group()
group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
- parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (only when using -x)')
+ parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")')
+ parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true")
parser_modify.set_defaults(func=modify)
parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe',
description='Extracts the source for an existing recipe',
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser_extract.add_argument('recipename', help='Name for recipe to extract the source for')
+ group='advanced')
+ parser_extract.add_argument('recipename', help='Name of recipe to extract the source for')
parser_extract.add_argument('srctree', help='Path to where to extract the source tree')
- parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
+ parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")')
parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
- parser_extract.set_defaults(func=extract)
+ parser_extract.set_defaults(func=extract, no_workspace=True)
+
+ parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe',
+ description='Synchronize the previously extracted source tree for an existing recipe',
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ group='advanced')
+ parser_sync.add_argument('recipename', help='Name of recipe to sync the source for')
+ parser_sync.add_argument('srctree', help='Path to the source tree')
+ parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
+ parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
+ parser_sync.set_defaults(func=sync)
+
+ parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace',
+ description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.',
+ group='working', order=10)
+ parser_rename.add_argument('recipename', help='Current name of recipe to rename')
+ parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)')
+ parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)')
+ parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path')
+ parser_rename.set_defaults(func=rename)
parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe',
- description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV)')
+ description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.',
+ group='working', order=-90)
parser_update_recipe.add_argument('recipename', help='Name of recipe to update')
parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
- parser_update_recipe.add_argument('--initial-rev', help='Starting revision for patches')
+ parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches')
+ parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR')
+ parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true')
parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
parser_update_recipe.set_defaults(func=update_recipe)
parser_status = subparsers.add_parser('status', help='Show workspace status',
description='Lists recipes currently in your workspace and the paths to their respective external source trees',
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ group='info', order=100)
parser_status.set_defaults(func=status)
- parser_build = subparsers.add_parser('build', help='Build a recipe',
- description='Builds the specified recipe using bitbake',
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser_build.add_argument('recipename', help='Recipe to build')
- parser_build.set_defaults(func=build)
-
parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace',
- description='Removes the specified recipe from your workspace (resetting its state)',
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser_reset.add_argument('recipename', nargs='?', help='Recipe to reset')
+ description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).',
+ group='working', order=-100)
+ parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset')
parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)')
parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output')
parser_reset.set_defaults(func=reset)
+
+ parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace',
+ description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified.',
+ group='working', order=-100)
+ parser_finish.add_argument('recipename', help='Recipe to finish')
+ parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.')
+ parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
+ parser_finish.add_argument('--initial-rev', help='Override starting revision for patches')
+ parser_finish.set_defaults(func=finish)
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
new file mode 100644
index 0000000000..05fb9e5ed0
--- /dev/null
+++ b/scripts/lib/devtool/upgrade.py
@@ -0,0 +1,409 @@
+# Development tool - upgrade command plugin
+#
+# Copyright (C) 2014-2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+"""Devtool upgrade plugin"""
+
+import os
+import sys
+import re
+import shutil
+import tempfile
+import logging
+import argparse
+import scriptutils
+import errno
+import bb
+
+devtool_path = os.path.dirname(os.path.realpath(__file__)) + '/../../../meta/lib'
+sys.path = sys.path + [devtool_path]
+
+import oe.recipeutils
+from devtool import standard
+from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe, use_external_build
+
+logger = logging.getLogger('devtool')
+
+def _run(cmd, cwd=''):
+ logger.debug("Running command %s> %s" % (cwd,cmd))
+ return bb.process.run('%s' % cmd, cwd=cwd)
+
+def _get_srctree(tmpdir):
+ srctree = tmpdir
+ dirs = os.listdir(tmpdir)
+ if len(dirs) == 1:
+ srctree = os.path.join(tmpdir, dirs[0])
+ return srctree
+
+def _copy_source_code(orig, dest):
+ for path in standard._ls_tree(orig):
+ dest_dir = os.path.join(dest, os.path.dirname(path))
+ bb.utils.mkdirhier(dest_dir)
+ dest_path = os.path.join(dest, path)
+ shutil.move(os.path.join(orig, path), dest_path)
+
+def _get_checksums(rf):
+ import re
+ checksums = {}
+ with open(rf) as f:
+ for line in f:
+ for cs in ['md5sum', 'sha256sum']:
+ m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line)
+ if m:
+ checksums[cs] = m.group(1)
+ return checksums
+
+def _remove_patch_dirs(recipefolder):
+ for root, dirs, files in os.walk(recipefolder):
+ for d in dirs:
+ shutil.rmtree(os.path.join(root,d))
+
+def _recipe_contains(rd, var):
+ rf = rd.getVar('FILE')
+ varfiles = oe.recipeutils.get_var_files(rf, [var], rd)
+ for var, fn in varfiles.items():
+ if fn and fn.startswith(os.path.dirname(rf) + os.sep):
+ return True
+ return False
+
+def _rename_recipe_dirs(oldpv, newpv, path):
+ for root, dirs, files in os.walk(path):
+ # Rename directories with the version in their name
+ for olddir in dirs:
+ if olddir.find(oldpv) != -1:
+ newdir = olddir.replace(oldpv, newpv)
+ if olddir != newdir:
+ shutil.move(os.path.join(path, olddir), os.path.join(path, newdir))
+ # Rename any inc files with the version in their name (unusual, but possible)
+ for oldfile in files:
+ if oldfile.endswith('.inc'):
+ if oldfile.find(oldpv) != -1:
+ newfile = oldfile.replace(oldpv, newpv)
+ if oldfile != newfile:
+ os.rename(os.path.join(path, oldfile), os.path.join(path, newfile))
+
+def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path):
+ oldrecipe = os.path.basename(oldrecipe)
+ if oldrecipe.endswith('_%s.bb' % oldpv):
+ newrecipe = '%s_%s.bb' % (bpn, newpv)
+ if oldrecipe != newrecipe:
+ shutil.move(os.path.join(path, oldrecipe), os.path.join(path, newrecipe))
+ else:
+ newrecipe = oldrecipe
+ return os.path.join(path, newrecipe)
+
+def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path):
+ _rename_recipe_dirs(oldpv, newpv, path)
+ return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path)
+
+def _write_append(rc, srctree, same_dir, no_same_dir, rev, copied, workspace, d):
+ """Writes an append file"""
+ if not os.path.exists(rc):
+ raise DevtoolError("bbappend not created because %s does not exist" % rc)
+
+ appendpath = os.path.join(workspace, 'appends')
+ if not os.path.exists(appendpath):
+ bb.utils.mkdirhier(appendpath)
+
+ brf = os.path.basename(os.path.splitext(rc)[0]) # rc basename
+
+ srctree = os.path.abspath(srctree)
+ pn = d.getVar('PN')
+ af = os.path.join(appendpath, '%s.bbappend' % brf)
+ with open(af, 'w') as f:
+ f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
+ f.write('inherit externalsrc\n')
+ f.write(('# NOTE: We use pn- overrides here to avoid affecting'
+ 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n'))
+ f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree))
+ b_is_s = use_external_build(same_dir, no_same_dir, d)
+ if b_is_s:
+ f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree))
+ f.write('\n')
+ if rev:
+ f.write('# initial_rev: %s\n' % rev)
+ if copied:
+ f.write('# original_path: %s\n' % os.path.dirname(d.getVar('FILE')))
+ f.write('# original_files: %s\n' % ' '.join(copied))
+ return af
+
+def _cleanup_on_error(rf, srctree):
+ rfp = os.path.split(rf)[0] # recipe folder
+ rfpp = os.path.split(rfp)[0] # recipes folder
+ if os.path.exists(rfp):
+ shutil.rmtree(b)
+ if not len(os.listdir(rfpp)):
+ os.rmdir(rfpp)
+ srctree = os.path.abspath(srctree)
+ if os.path.exists(srctree):
+ shutil.rmtree(srctree)
+
+def _upgrade_error(e, rf, srctree):
+ if rf:
+ cleanup_on_error(rf, srctree)
+ logger.error(e)
+ raise DevtoolError(e)
+
+def _get_uri(rd):
+ srcuris = rd.getVar('SRC_URI').split()
+ if not len(srcuris):
+ raise DevtoolError('SRC_URI not found on recipe')
+ # Get first non-local entry in SRC_URI - usually by convention it's
+ # the first entry, but not always!
+ srcuri = None
+ for entry in srcuris:
+ if not entry.startswith('file://'):
+ srcuri = entry
+ break
+ if not srcuri:
+ raise DevtoolError('Unable to find non-local entry in SRC_URI')
+ srcrev = '${AUTOREV}'
+ if '://' in srcuri:
+ # Fetch a URL
+ rev_re = re.compile(';rev=([^;]+)')
+ res = rev_re.search(srcuri)
+ if res:
+ srcrev = res.group(1)
+ srcuri = rev_re.sub('', srcuri)
+ return srcuri, srcrev
+
+def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tinfoil, rd):
+ """Extract sources of a recipe with a new version"""
+
+ def __run(cmd):
+ """Simple wrapper which calls _run with srctree as cwd"""
+ return _run(cmd, srctree)
+
+ crd = rd.createCopy()
+
+ pv = crd.getVar('PV')
+ crd.setVar('PV', newpv)
+
+ tmpsrctree = None
+ uri, rev = _get_uri(crd)
+ if srcrev:
+ rev = srcrev
+ if uri.startswith('git://'):
+ __run('git fetch')
+ __run('git checkout %s' % rev)
+ __run('git tag -f devtool-base-new')
+ md5 = None
+ sha256 = None
+ else:
+ __run('git checkout devtool-base -b devtool-%s' % newpv)
+
+ tmpdir = tempfile.mkdtemp(prefix='devtool')
+ try:
+ md5, sha256 = scriptutils.fetch_uri(tinfoil.config_data, uri, tmpdir, rev)
+ except bb.fetch2.FetchError as e:
+ raise DevtoolError(e)
+
+ tmpsrctree = _get_srctree(tmpdir)
+ srctree = os.path.abspath(srctree)
+
+ # Delete all sources so we ensure no stray files are left over
+ for item in os.listdir(srctree):
+ if item in ['.git', 'oe-local-files']:
+ continue
+ itempath = os.path.join(srctree, item)
+ if os.path.isdir(itempath):
+ shutil.rmtree(itempath)
+ else:
+ os.remove(itempath)
+
+ # Copy in new ones
+ _copy_source_code(tmpsrctree, srctree)
+
+ (stdout,_) = __run('git ls-files --modified --others --exclude-standard')
+ for f in stdout.splitlines():
+ __run('git add "%s"' % f)
+
+ useroptions = []
+ oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd)
+ __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv))
+ __run('git tag -f devtool-base-%s' % newpv)
+
+ (stdout, _) = __run('git rev-parse HEAD')
+ rev = stdout.rstrip()
+
+ if no_patch:
+ patches = oe.recipeutils.get_recipe_patches(crd)
+ if len(patches):
+ logger.warn('By user choice, the following patches will NOT be applied')
+ for patch in patches:
+ logger.warn("%s" % os.path.basename(patch))
+ else:
+ __run('git checkout devtool-patched -b %s' % branch)
+ skiptag = False
+ try:
+ __run('git rebase %s' % rev)
+ except bb.process.ExecutionError as e:
+ skiptag = True
+ if 'conflict' in e.stdout:
+ logger.warn('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip()))
+ else:
+ logger.warn('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
+ if not skiptag:
+ if uri.startswith('git://'):
+ suffix = 'new'
+ else:
+ suffix = newpv
+ __run('git tag -f devtool-patched-%s' % suffix)
+
+ if tmpsrctree:
+ if keep_temp:
+ logger.info('Preserving temporary directory %s' % tmpsrctree)
+ else:
+ shutil.rmtree(tmpsrctree)
+
+ return (rev, md5, sha256)
+
+def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, workspace, tinfoil, rd):
+ """Creates the new recipe under workspace"""
+
+ bpn = rd.getVar('BPN')
+ path = os.path.join(workspace, 'recipes', bpn)
+ bb.utils.mkdirhier(path)
+ copied, _ = oe.recipeutils.copy_recipe_files(rd, path)
+
+ oldpv = rd.getVar('PV')
+ if not newpv:
+ newpv = oldpv
+ origpath = rd.getVar('FILE')
+ fullpath = _rename_recipe_files(origpath, bpn, oldpv, newpv, path)
+ logger.debug('Upgraded %s => %s' % (origpath, fullpath))
+
+ newvalues = {}
+ if _recipe_contains(rd, 'PV') and newpv != oldpv:
+ newvalues['PV'] = newpv
+
+ if srcrev:
+ newvalues['SRCREV'] = srcrev
+
+ if srcbranch:
+ src_uri = oe.recipeutils.split_var_value(rd.getVar('SRC_URI', False) or '')
+ changed = False
+ replacing = True
+ new_src_uri = []
+ for entry in src_uri:
+ scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry)
+ if replacing and scheme in ['git', 'gitsm']:
+ branch = params.get('branch', 'master')
+ if rd.expand(branch) != srcbranch:
+ # Handle case where branch is set through a variable
+ res = re.match(r'\$\{([^}@]+)\}', branch)
+ if res:
+ newvalues[res.group(1)] = srcbranch
+ # We know we won't change SRC_URI now, so break out
+ break
+ else:
+ params['branch'] = srcbranch
+ entry = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params))
+ changed = True
+ replacing = False
+ new_src_uri.append(entry)
+ if changed:
+ newvalues['SRC_URI'] = ' '.join(new_src_uri)
+
+ newvalues['PR'] = None
+
+ if md5 and sha256:
+ newvalues['SRC_URI[md5sum]'] = md5
+ newvalues['SRC_URI[sha256sum]'] = sha256
+
+ rd = tinfoil.parse_recipe_file(fullpath, False)
+ oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
+
+ return fullpath, copied
+
+def upgrade(args, config, basepath, workspace):
+ """Entry point for the devtool 'upgrade' subcommand"""
+
+ if args.recipename in workspace:
+ raise DevtoolError("recipe %s is already in your workspace" % args.recipename)
+ if not args.version and not args.srcrev:
+ raise DevtoolError("You must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option")
+ if args.srcbranch and not args.srcrev:
+ raise DevtoolError("If you specify --srcbranch/-B then you must use --srcrev/-S to specify the revision" % args.recipename)
+
+ tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
+ try:
+ rd = parse_recipe(config, tinfoil, args.recipename, True)
+ if not rd:
+ return 1
+
+ pn = rd.getVar('PN')
+ if pn != args.recipename:
+ logger.info('Mapping %s to %s' % (args.recipename, pn))
+ if pn in workspace:
+ raise DevtoolError("recipe %s is already in your workspace" % pn)
+
+ if args.srctree:
+ srctree = os.path.abspath(args.srctree)
+ else:
+ srctree = standard.get_default_srctree(config, pn)
+
+ standard._check_compatible_recipe(pn, rd)
+ old_srcrev = rd.getVar('SRCREV')
+ if old_srcrev == 'INVALID':
+ old_srcrev = None
+ if old_srcrev and not args.srcrev:
+ raise DevtoolError("Recipe specifies a SRCREV value; you must specify a new one when upgrading")
+ if rd.getVar('PV') == args.version and old_srcrev == args.srcrev:
+ raise DevtoolError("Current and upgrade versions are the same version")
+
+ rf = None
+ try:
+ rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, rd, tinfoil)
+ rev2, md5, sha256 = _extract_new_source(args.version, srctree, args.no_patch,
+ args.srcrev, args.branch, args.keep_temp,
+ tinfoil, rd)
+ rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, args.srcbranch, config.workspace_path, tinfoil, rd)
+ except bb.process.CmdError as e:
+ _upgrade_error(e, rf, srctree)
+ except DevtoolError as e:
+ _upgrade_error(e, rf, srctree)
+ standard._add_md5(config, pn, os.path.dirname(rf))
+
+ af = _write_append(rf, srctree, args.same_dir, args.no_same_dir, rev2,
+ copied, config.workspace_path, rd)
+ standard._add_md5(config, pn, af)
+ logger.info('Upgraded source extracted to %s' % srctree)
+ logger.info('New recipe is %s' % rf)
+ finally:
+ tinfoil.shutdown()
+ return 0
+
+def register_commands(subparsers, context):
+ """Register devtool subcommands from this plugin"""
+
+ defsrctree = standard.get_default_srctree(context.config)
+
+ parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe',
+ description='Upgrades an existing recipe to a new upstream version. Puts the upgraded recipe file into the workspace along with any associated files, and extracts the source tree to a specified location (in case patches need rebasing or adding to as a result of the upgrade).',
+ group='starting')
+ parser_upgrade.add_argument('recipename', help='Name of recipe to upgrade (just name - no version, path or extension)')
+ parser_upgrade.add_argument('srctree', nargs='?', help='Path to where to extract the source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
+ parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV)')
+ parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (required if fetching from an SCM such as git)')
+ parser_upgrade.add_argument('--srcbranch', '-B', help='Branch in source repository containing the revision to use (if fetching from an SCM such as git)')
+ parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")')
+ parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code')
+ group = parser_upgrade.add_mutually_exclusive_group()
+ group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
+ group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
+ parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
+ parser_upgrade.set_defaults(func=upgrade)
diff --git a/scripts/lib/devtool/utilcmds.py b/scripts/lib/devtool/utilcmds.py
new file mode 100644
index 0000000000..0437e6417c
--- /dev/null
+++ b/scripts/lib/devtool/utilcmds.py
@@ -0,0 +1,233 @@
+# Development tool - utility commands plugin
+#
+# Copyright (C) 2015-2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+"""Devtool utility plugins"""
+
+import os
+import sys
+import shutil
+import tempfile
+import logging
+import argparse
+import subprocess
+import scriptutils
+from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, DevtoolError
+from devtool import parse_recipe
+
+logger = logging.getLogger('devtool')
+
+
+def edit_recipe(args, config, basepath, workspace):
+ """Entry point for the devtool 'edit-recipe' subcommand"""
+ if args.any_recipe:
+ tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
+ try:
+ rd = parse_recipe(config, tinfoil, args.recipename, True)
+ if not rd:
+ return 1
+ recipefile = rd.getVar('FILE')
+ finally:
+ tinfoil.shutdown()
+ else:
+ check_workspace_recipe(workspace, args.recipename)
+ recipefile = workspace[args.recipename]['recipefile']
+ if not recipefile:
+ raise DevtoolError("Recipe file for %s is not under the workspace" %
+ args.recipename)
+
+ return scriptutils.run_editor(recipefile)
+
+
+def configure_help(args, config, basepath, workspace):
+ """Entry point for the devtool 'configure-help' subcommand"""
+ import oe.utils
+
+ check_workspace_recipe(workspace, args.recipename)
+ tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
+ try:
+ rd = parse_recipe(config, tinfoil, args.recipename, appends=True, filter_workspace=False)
+ if not rd:
+ return 1
+ b = rd.getVar('B')
+ s = rd.getVar('S')
+ configurescript = os.path.join(s, 'configure')
+ confdisabled = 'noexec' in rd.getVarFlags('do_configure') or 'do_configure' not in (rd.getVar('__BBTASKS', False) or [])
+ configureopts = oe.utils.squashspaces(rd.getVar('CONFIGUREOPTS') or '')
+ extra_oeconf = oe.utils.squashspaces(rd.getVar('EXTRA_OECONF') or '')
+ extra_oecmake = oe.utils.squashspaces(rd.getVar('EXTRA_OECMAKE') or '')
+ do_configure = rd.getVar('do_configure') or ''
+ do_configure_noexpand = rd.getVar('do_configure', False) or ''
+ packageconfig = rd.getVarFlags('PACKAGECONFIG') or []
+ autotools = bb.data.inherits_class('autotools', rd) and ('oe_runconf' in do_configure or 'autotools_do_configure' in do_configure)
+ cmake = bb.data.inherits_class('cmake', rd) and ('cmake_do_configure' in do_configure)
+ cmake_do_configure = rd.getVar('cmake_do_configure')
+ pn = rd.getVar('PN')
+ finally:
+ tinfoil.shutdown()
+
+ if 'doc' in packageconfig:
+ del packageconfig['doc']
+
+ if autotools and not os.path.exists(configurescript):
+ logger.info('Running do_configure to generate configure script')
+ try:
+ stdout, _ = exec_build_env_command(config.init_path, basepath,
+ 'bitbake -c configure %s' % args.recipename,
+ stderr=subprocess.STDOUT)
+ except bb.process.ExecutionError:
+ pass
+
+ if confdisabled or do_configure.strip() in ('', ':'):
+ raise DevtoolError("do_configure task has been disabled for this recipe")
+ elif args.no_pager and not os.path.exists(configurescript):
+ raise DevtoolError("No configure script found and no other information to display")
+ else:
+ configopttext = ''
+ if autotools and configureopts:
+ configopttext = '''
+Arguments currently passed to the configure script:
+
+%s
+
+Some of those are fixed.''' % (configureopts + ' ' + extra_oeconf)
+ if extra_oeconf:
+ configopttext += ''' The ones that are specified through EXTRA_OECONF (which you can change or add to easily):
+
+%s''' % extra_oeconf
+
+ elif cmake:
+ in_cmake = False
+ cmake_cmd = ''
+ for line in cmake_do_configure.splitlines():
+ if in_cmake:
+ cmake_cmd = cmake_cmd + ' ' + line.strip().rstrip('\\')
+ if not line.endswith('\\'):
+ break
+ if line.lstrip().startswith('cmake '):
+ cmake_cmd = line.strip().rstrip('\\')
+ if line.endswith('\\'):
+ in_cmake = True
+ else:
+ break
+ if cmake_cmd:
+ configopttext = '''
+The current cmake command line:
+
+%s
+
+Arguments specified through EXTRA_OECMAKE (which you can change or add to easily)
+
+%s''' % (oe.utils.squashspaces(cmake_cmd), extra_oecmake)
+ else:
+ configopttext = '''
+The current implementation of cmake_do_configure:
+
+cmake_do_configure() {
+%s
+}
+
+Arguments specified through EXTRA_OECMAKE (which you can change or add to easily)
+
+%s''' % (cmake_do_configure.rstrip(), extra_oecmake)
+
+ elif do_configure:
+ configopttext = '''
+The current implementation of do_configure:
+
+do_configure() {
+%s
+}''' % do_configure.rstrip()
+ if '${EXTRA_OECONF}' in do_configure_noexpand:
+ configopttext += '''
+
+Arguments specified through EXTRA_OECONF (which you can change or add to easily):
+
+%s''' % extra_oeconf
+
+ if packageconfig:
+ configopttext += '''
+
+Some of these options may be controlled through PACKAGECONFIG; for more details please see the recipe.'''
+
+ if args.arg:
+ helpargs = ' '.join(args.arg)
+ elif cmake:
+ helpargs = '-LH'
+ else:
+ helpargs = '--help'
+
+ msg = '''configure information for %s
+------------------------------------------
+%s''' % (pn, configopttext)
+
+ if cmake:
+ msg += '''
+
+The cmake %s output for %s follows. After "-- Cache values" you should see a list of variables you can add to EXTRA_OECMAKE (prefixed with -D and suffixed with = followed by the desired value, without any spaces).
+------------------------------------------''' % (helpargs, pn)
+ elif os.path.exists(configurescript):
+ msg += '''
+
+The ./configure %s output for %s follows.
+------------------------------------------''' % (helpargs, pn)
+
+ olddir = os.getcwd()
+ tmppath = tempfile.mkdtemp()
+ with tempfile.NamedTemporaryFile('w', delete=False) as tf:
+ if not args.no_header:
+ tf.write(msg + '\n')
+ tf.close()
+ try:
+ try:
+ cmd = 'cat %s' % tf.name
+ if cmake:
+ cmd += '; cmake %s %s 2>&1' % (helpargs, s)
+ os.chdir(b)
+ elif os.path.exists(configurescript):
+ cmd += '; %s %s' % (configurescript, helpargs)
+ if sys.stdout.isatty() and not args.no_pager:
+ pager = os.environ.get('PAGER', 'less')
+ cmd = '(%s) | %s' % (cmd, pager)
+ subprocess.check_call(cmd, shell=True)
+ except subprocess.CalledProcessError as e:
+ return e.returncode
+ finally:
+ os.chdir(olddir)
+ shutil.rmtree(tmppath)
+ os.remove(tf.name)
+
+
+def register_commands(subparsers, context):
+ """Register devtool subcommands from this plugin"""
+ parser_edit_recipe = subparsers.add_parser('edit-recipe', help='Edit a recipe file in your workspace',
+ description='Runs the default editor (as specified by the EDITOR variable) on the specified recipe. Note that the recipe file itself must be in the workspace (i.e. as a result of "devtool add" or "devtool upgrade"); you can override this with the -a/--any-recipe option.',
+ group='working')
+ parser_edit_recipe.add_argument('recipename', help='Recipe to edit')
+ parser_edit_recipe.add_argument('--any-recipe', '-a', action="store_true", help='Edit any recipe, not just where the recipe file itself is in the workspace')
+ parser_edit_recipe.set_defaults(func=edit_recipe)
+
+ # NOTE: Needed to override the usage string here since the default
+ # gets the order wrong - recipename must come before --arg
+ parser_configure_help = subparsers.add_parser('configure-help', help='Get help on configure script options',
+ usage='devtool configure-help [options] recipename [--arg ...]',
+ description='Displays the help for the configure script for the specified recipe (i.e. runs ./configure --help) prefaced by a header describing the current options being specified. Output is piped through less (or whatever PAGER is set to, if set) for easy browsing.',
+ group='working')
+ parser_configure_help.add_argument('recipename', help='Recipe to show configure help for')
+ parser_configure_help.add_argument('-p', '--no-pager', help='Disable paged output', action="store_true")
+ parser_configure_help.add_argument('-n', '--no-header', help='Disable explanatory header text', action="store_true")
+ parser_configure_help.add_argument('--arg', help='Pass remaining arguments to the configure script instead of --help (useful if the script has additional help options)', nargs=argparse.REMAINDER)
+ parser_configure_help.set_defaults(func=configure_help)
diff --git a/scripts/lib/image/__init__.py b/scripts/lib/image/__init__.py
deleted file mode 100644
index 1ff814e761..0000000000
--- a/scripts/lib/image/__init__.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# OpenEmbedded Image tools library
-#
-# Copyright (c) 2013, Intel Corporation.
-# All rights reserved.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# AUTHORS
-# Tom Zanussi <tom.zanussi (at] linux.intel.com>
-#
diff --git a/scripts/lib/image/canned-wks/mkgummidisk.wks b/scripts/lib/image/canned-wks/mkgummidisk.wks
deleted file mode 100644
index f81cbdfb84..0000000000
--- a/scripts/lib/image/canned-wks/mkgummidisk.wks
+++ /dev/null
@@ -1,11 +0,0 @@
-# short-description: Create an EFI disk image
-# long-description: Creates a partitioned EFI disk image that the user
-# can directly dd to boot media.
-
-part /boot --source bootimg-efi --sourceparams="loader=gummiboot" --ondisk sda --label msdos --active --align 1024
-
-part / --source rootfs --ondisk sda --fstype=ext3 --label platform --align 1024
-
-part swap --ondisk sda --size 44 --label swap1 --fstype=swap
-
-bootloader --timeout=10 --append="rootwait rootfstype=ext3 console=ttyPCH0,115200 console=tty0 vmalloc=256MB snd-hda-intel.enable_msi=0"
diff --git a/scripts/lib/image/config/wic.conf b/scripts/lib/image/config/wic.conf
deleted file mode 100644
index a51bcb55eb..0000000000
--- a/scripts/lib/image/config/wic.conf
+++ /dev/null
@@ -1,6 +0,0 @@
-[common]
-; general settings
-distro_name = OpenEmbedded
-
-[create]
-; settings for create subcommand
diff --git a/scripts/lib/image/engine.py b/scripts/lib/image/engine.py
deleted file mode 100644
index 68d1ce2659..0000000000
--- a/scripts/lib/image/engine.py
+++ /dev/null
@@ -1,279 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (c) 2013, Intel Corporation.
-# All rights reserved.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# DESCRIPTION
-
-# This module implements the image creation engine used by 'wic' to
-# create images. The engine parses through the OpenEmbedded kickstart
-# (wks) file specified and generates images that can then be directly
-# written onto media.
-#
-# AUTHORS
-# Tom Zanussi <tom.zanussi (at] linux.intel.com>
-#
-
-import os
-import sys
-from abc import ABCMeta, abstractmethod
-import shlex
-import json
-import subprocess
-import shutil
-
-import os, sys, errno
-from wic import msger, creator
-from wic.utils import cmdln, misc, errors
-from wic.conf import configmgr
-from wic.plugin import pluginmgr
-from wic.__version__ import VERSION
-from wic.utils.oe.misc import *
-
-
-def verify_build_env():
- """
- Verify that the build environment is sane.
-
- Returns True if it is, false otherwise
- """
- try:
- builddir = os.environ["BUILDDIR"]
- except KeyError:
- print "BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)"
- sys.exit(1)
-
- return True
-
-
-def find_artifacts(image_name):
- """
- Gather the build artifacts for the current image (the image_name
- e.g. core-image-minimal) for the current MACHINE set in local.conf
- """
- bitbake_env_lines = get_bitbake_env_lines()
-
- rootfs_dir = kernel_dir = bootimg_dir = native_sysroot = ""
-
- for line in bitbake_env_lines.split('\n'):
- if (get_line_val(line, "IMAGE_ROOTFS")):
- rootfs_dir = get_line_val(line, "IMAGE_ROOTFS")
- continue
- if (get_line_val(line, "DEPLOY_DIR_IMAGE")):
- kernel_dir = get_line_val(line, "DEPLOY_DIR_IMAGE")
- continue
- if (get_line_val(line, "STAGING_DIR_NATIVE")):
- native_sysroot = get_line_val(line, "STAGING_DIR_NATIVE")
- continue
-
- return (rootfs_dir, kernel_dir, bootimg_dir, native_sysroot)
-
-
-CANNED_IMAGE_DIR = "lib/image/canned-wks" # relative to scripts
-SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR
-
-def build_canned_image_list(dl):
- layers_path = get_bitbake_var("BBLAYERS")
- canned_wks_layer_dirs = []
-
- if layers_path is not None:
- for layer_path in layers_path.split():
- path = os.path.join(layer_path, SCRIPTS_CANNED_IMAGE_DIR)
- canned_wks_layer_dirs.append(path)
-
- path = os.path.join(dl, CANNED_IMAGE_DIR)
- canned_wks_layer_dirs.append(path)
-
- return canned_wks_layer_dirs
-
-def find_canned_image(scripts_path, wks_file):
- """
- Find a .wks file with the given name in the canned files dir.
-
- Return False if not found
- """
- layers_canned_wks_dir = build_canned_image_list(scripts_path)
-
- for canned_wks_dir in layers_canned_wks_dir:
- for root, dirs, files in os.walk(canned_wks_dir):
- for file in files:
- if file.endswith("~") or file.endswith("#"):
- continue
- if file.endswith(".wks") and wks_file + ".wks" == file:
- fullpath = os.path.join(canned_wks_dir, file)
- return fullpath
- return None
-
-
-def list_canned_images(scripts_path):
- """
- List the .wks files in the canned image dir, minus the extension.
- """
- layers_canned_wks_dir = build_canned_image_list(scripts_path)
-
- for canned_wks_dir in layers_canned_wks_dir:
- for root, dirs, files in os.walk(canned_wks_dir):
- for file in files:
- if file.endswith("~") or file.endswith("#"):
- continue
- if file.endswith(".wks"):
- fullpath = os.path.join(canned_wks_dir, file)
- f = open(fullpath, "r")
- lines = f.readlines()
- for line in lines:
- desc = ""
- idx = line.find("short-description:")
- if idx != -1:
- desc = line[idx + len("short-description:"):].strip()
- break
- basename = os.path.splitext(file)[0]
- print " %s\t\t%s" % (basename.ljust(30), desc)
-
-
-def list_canned_image_help(scripts_path, fullpath):
- """
- List the help and params in the specified canned image.
- """
- f = open(fullpath, "r")
- lines = f.readlines()
- found = False
- for line in lines:
- if not found:
- idx = line.find("long-description:")
- if idx != -1:
- print
- print line[idx + len("long-description:"):].strip()
- found = True
- continue
- if not line.strip():
- break
- idx = line.find("#")
- if idx != -1:
- print line[idx + len("#:"):].rstrip()
- else:
- break
-
-
-def list_source_plugins():
- """
- List the available source plugins i.e. plugins available for --source.
- """
- plugins = pluginmgr.get_source_plugins()
-
- for plugin in plugins:
- print " %s" % plugin
-
-
-def wic_create(args, wks_file, rootfs_dir, bootimg_dir, kernel_dir,
- native_sysroot, scripts_path, image_output_dir, debug,
- properties_file, properties=None):
- """Create image
-
- wks_file - user-defined OE kickstart file
- rootfs_dir - absolute path to the build's /rootfs dir
- bootimg_dir - absolute path to the build's boot artifacts directory
- kernel_dir - absolute path to the build's kernel directory
- native_sysroot - absolute path to the build's native sysroots dir
- scripts_path - absolute path to /scripts dir
- image_output_dir - dirname to create for image
- properties_file - use values from this file if nonempty i.e no prompting
- properties - use values from this string if nonempty i.e no prompting
-
- Normally, the values for the build artifacts values are determined
- by 'wic -e' from the output of the 'bitbake -e' command given an
- image name e.g. 'core-image-minimal' and a given machine set in
- local.conf. If that's the case, the variables get the following
- values from the output of 'bitbake -e':
-
- rootfs_dir: IMAGE_ROOTFS
- kernel_dir: DEPLOY_DIR_IMAGE
- native_sysroot: STAGING_DIR_NATIVE
-
- In the above case, bootimg_dir remains unset and the
- plugin-specific image creation code is responsible for finding the
- bootimg artifacts.
-
- In the case where the values are passed in explicitly i.e 'wic -e'
- is not used but rather the individual 'wic' options are used to
- explicitly specify these values.
- """
- try:
- oe_builddir = os.environ["BUILDDIR"]
- except KeyError:
- print "BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)"
- sys.exit(1)
-
- direct_args = list()
- direct_args.insert(0, oe_builddir)
- direct_args.insert(0, image_output_dir)
- direct_args.insert(0, wks_file)
- direct_args.insert(0, rootfs_dir)
- direct_args.insert(0, bootimg_dir)
- direct_args.insert(0, kernel_dir)
- direct_args.insert(0, native_sysroot)
- direct_args.insert(0, "direct")
-
- if debug:
- msger.set_loglevel('debug')
-
- cr = creator.Creator()
-
- cr.main(direct_args)
-
- print "\nThe image(s) were created using OE kickstart file:\n %s" % wks_file
-
-
-def wic_list(args, scripts_path, properties_file):
- """
- Print the complete list of properties defined by the image, or the
- possible values for a particular image property.
- """
- if len(args) < 1:
- return False
-
- if len(args) == 1:
- if args[0] == "images":
- list_canned_images(scripts_path)
- return True
- elif args[0] == "source-plugins":
- list_source_plugins()
- return True
- elif args[0] == "properties":
- return True
- else:
- return False
-
- if len(args) == 2:
- if args[0] == "properties":
- wks_file = args[1]
- print "print properties contained in wks file: %s" % wks_file
- return True
- elif args[0] == "property":
- print "print property values for property: %s" % args[1]
- return True
- elif args[1] == "help":
- wks_file = args[0]
- fullpath = find_canned_image(scripts_path, wks_file)
- if not fullpath:
- print "No image named %s found, exiting. (Use 'wic list images' to list available images, or specify a fully-qualified OE kickstart (.wks) filename)\n" % wks_file
- sys.exit(1)
- list_canned_image_help(scripts_path, fullpath)
- return True
- else:
- return False
-
- return False
diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py
new file mode 100644
index 0000000000..69c8bb77a0
--- /dev/null
+++ b/scripts/lib/recipetool/append.py
@@ -0,0 +1,457 @@
+# Recipe creation tool - append plugin
+#
+# Copyright (C) 2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import sys
+import os
+import argparse
+import glob
+import fnmatch
+import re
+import subprocess
+import logging
+import stat
+import shutil
+import scriptutils
+import errno
+from collections import defaultdict
+
+logger = logging.getLogger('recipetool')
+
+tinfoil = None
+
+def tinfoil_init(instance):
+ global tinfoil
+ tinfoil = instance
+
+
+# FIXME guessing when we don't have pkgdata?
+# FIXME mode to create patch rather than directly substitute
+
+class InvalidTargetFileError(Exception):
+ pass
+
+def find_target_file(targetpath, d, pkglist=None):
+ """Find the recipe installing the specified target path, optionally limited to a select list of packages"""
+ import json
+
+ pkgdata_dir = d.getVar('PKGDATA_DIR')
+
+ # The mix between /etc and ${sysconfdir} here may look odd, but it is just
+ # being consistent with usage elsewhere
+ invalidtargets = {'${sysconfdir}/version': '${sysconfdir}/version is written out at image creation time',
+ '/etc/timestamp': '/etc/timestamp is written out at image creation time',
+ '/dev/*': '/dev is handled by udev (or equivalent) and the kernel (devtmpfs)',
+ '/etc/passwd': '/etc/passwd should be managed through the useradd and extrausers classes',
+ '/etc/group': '/etc/group should be managed through the useradd and extrausers classes',
+ '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes',
+ '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes',
+ '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname_pn-base-files = "value" in configuration',}
+
+ for pthspec, message in invalidtargets.items():
+ if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)):
+ raise InvalidTargetFileError(d.expand(message))
+
+ targetpath_re = re.compile(r'\s+(\$D)?%s(\s|$)' % targetpath)
+
+ recipes = defaultdict(list)
+ for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')):
+ if pkglist:
+ filelist = pkglist
+ else:
+ filelist = files
+ for fn in filelist:
+ pkgdatafile = os.path.join(root, fn)
+ if pkglist and not os.path.exists(pkgdatafile):
+ continue
+ with open(pkgdatafile, 'r') as f:
+ pn = ''
+ # This does assume that PN comes before other values, but that's a fairly safe assumption
+ for line in f:
+ if line.startswith('PN:'):
+ pn = line.split(':', 1)[1].strip()
+ elif line.startswith('FILES_INFO:'):
+ val = line.split(':', 1)[1].strip()
+ dictval = json.loads(val)
+ for fullpth in dictval.keys():
+ if fnmatch.fnmatchcase(fullpth, targetpath):
+ recipes[targetpath].append(pn)
+ elif line.startswith('pkg_preinst_') or line.startswith('pkg_postinst_'):
+ scriptval = line.split(':', 1)[1].strip().encode('utf-8').decode('unicode_escape')
+ if 'update-alternatives --install %s ' % targetpath in scriptval:
+ recipes[targetpath].append('?%s' % pn)
+ elif targetpath_re.search(scriptval):
+ recipes[targetpath].append('!%s' % pn)
+ return recipes
+
+def _parse_recipe(pn, tinfoil):
+ try:
+ rd = tinfoil.parse_recipe(pn)
+ except bb.providers.NoProvider as e:
+ logger.error(str(e))
+ return None
+ return rd
+
+def determine_file_source(targetpath, rd):
+ """Assuming we know a file came from a specific recipe, figure out exactly where it came from"""
+ import oe.recipeutils
+
+ # See if it's in do_install for the recipe
+ workdir = rd.getVar('WORKDIR')
+ src_uri = rd.getVar('SRC_URI')
+ srcfile = ''
+ modpatches = []
+ elements = check_do_install(rd, targetpath)
+ if elements:
+ logger.debug('do_install line:\n%s' % ' '.join(elements))
+ srcpath = get_source_path(elements)
+ logger.debug('source path: %s' % srcpath)
+ if not srcpath.startswith('/'):
+ # Handle non-absolute path
+ srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath))
+ if srcpath.startswith(workdir):
+ # OK, now we have the source file name, look for it in SRC_URI
+ workdirfile = os.path.relpath(srcpath, workdir)
+ # FIXME this is where we ought to have some code in the fetcher, because this is naive
+ for item in src_uri.split():
+ localpath = bb.fetch2.localpath(item, rd)
+ # Source path specified in do_install might be a glob
+ if fnmatch.fnmatch(os.path.basename(localpath), workdirfile):
+ srcfile = 'file://%s' % localpath
+ elif '/' in workdirfile:
+ if item == 'file://%s' % workdirfile:
+ srcfile = 'file://%s' % localpath
+
+ # Check patches
+ srcpatches = []
+ patchedfiles = oe.recipeutils.get_recipe_patched_files(rd)
+ for patch, filelist in patchedfiles.items():
+ for fileitem in filelist:
+ if fileitem[0] == srcpath:
+ srcpatches.append((patch, fileitem[1]))
+ if srcpatches:
+ addpatch = None
+ for patch in srcpatches:
+ if patch[1] == 'A':
+ addpatch = patch[0]
+ else:
+ modpatches.append(patch[0])
+ if addpatch:
+ srcfile = 'patch://%s' % addpatch
+
+ return (srcfile, elements, modpatches)
+
+def get_source_path(cmdelements):
+ """Find the source path specified within a command"""
+ command = cmdelements[0]
+ if command in ['install', 'cp']:
+ helptext = subprocess.check_output('LC_ALL=C %s --help' % command, shell=True).decode('utf-8')
+ argopts = ''
+ argopt_line_re = re.compile('^-([a-zA-Z0-9]), --[a-z-]+=')
+ for line in helptext.splitlines():
+ line = line.lstrip()
+ res = argopt_line_re.search(line)
+ if res:
+ argopts += res.group(1)
+ if not argopts:
+ # Fallback
+ if command == 'install':
+ argopts = 'gmoSt'
+ elif command == 'cp':
+ argopts = 't'
+ else:
+ raise Exception('No fallback arguments for command %s' % command)
+
+ skipnext = False
+ for elem in cmdelements[1:-1]:
+ if elem.startswith('-'):
+ if len(elem) > 1 and elem[1] in argopts:
+ skipnext = True
+ continue
+ if skipnext:
+ skipnext = False
+ continue
+ return elem
+ else:
+ raise Exception('get_source_path: no handling for command "%s"')
+
+def get_func_deps(func, d):
+ """Find the function dependencies of a shell function"""
+ deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func))
+ deps |= set((d.getVarFlag(func, "vardeps") or "").split())
+ funcdeps = []
+ for dep in deps:
+ if d.getVarFlag(dep, 'func'):
+ funcdeps.append(dep)
+ return funcdeps
+
+def check_do_install(rd, targetpath):
+ """Look at do_install for a command that installs/copies the specified target path"""
+ instpath = os.path.abspath(os.path.join(rd.getVar('D'), targetpath.lstrip('/')))
+ do_install = rd.getVar('do_install')
+ # Handle where do_install calls other functions (somewhat crudely, but good enough for this purpose)
+ deps = get_func_deps('do_install', rd)
+ for dep in deps:
+ do_install = do_install.replace(dep, rd.getVar(dep))
+
+ # Look backwards through do_install as we want to catch where a later line (perhaps
+ # from a bbappend) is writing over the top
+ for line in reversed(do_install.splitlines()):
+ line = line.strip()
+ if (line.startswith('install ') and ' -m' in line) or line.startswith('cp '):
+ elements = line.split()
+ destpath = os.path.abspath(elements[-1])
+ if destpath == instpath:
+ return elements
+ elif destpath.rstrip('/') == os.path.dirname(instpath):
+ # FIXME this doesn't take recursive copy into account; unsure if it's practical to do so
+ srcpath = get_source_path(elements)
+ if fnmatch.fnmatchcase(os.path.basename(instpath), os.path.basename(srcpath)):
+ return elements
+ return None
+
+
+def appendfile(args):
+ import oe.recipeutils
+
+ stdout = ''
+ try:
+ (stdout, _) = bb.process.run('LANG=C file -b %s' % args.newfile, shell=True)
+ if 'cannot open' in stdout:
+ raise bb.process.ExecutionError(stdout)
+ except bb.process.ExecutionError as err:
+ logger.debug('file command returned error: %s' % err)
+ stdout = ''
+ if stdout:
+ logger.debug('file command output: %s' % stdout.rstrip())
+ if ('executable' in stdout and not 'shell script' in stdout) or 'shared object' in stdout:
+ logger.warn('This file looks like it is a binary or otherwise the output of compilation. If it is, you should consider building it properly instead of substituting a binary file directly.')
+
+ if args.recipe:
+ recipes = {args.targetpath: [args.recipe],}
+ else:
+ try:
+ recipes = find_target_file(args.targetpath, tinfoil.config_data)
+ except InvalidTargetFileError as e:
+ logger.error('%s cannot be handled by this tool: %s' % (args.targetpath, e))
+ return 1
+ if not recipes:
+ logger.error('Unable to find any package producing path %s - this may be because the recipe packaging it has not been built yet' % args.targetpath)
+ return 1
+
+ alternative_pns = []
+ postinst_pns = []
+
+ selectpn = None
+ for targetpath, pnlist in recipes.items():
+ for pn in pnlist:
+ if pn.startswith('?'):
+ alternative_pns.append(pn[1:])
+ elif pn.startswith('!'):
+ postinst_pns.append(pn[1:])
+ elif selectpn:
+ # hit here with multilibs
+ continue
+ else:
+ selectpn = pn
+
+ if not selectpn and len(alternative_pns) == 1:
+ selectpn = alternative_pns[0]
+ logger.error('File %s is an alternative possibly provided by recipe %s but seemingly no other, selecting it by default - you should double check other recipes' % (args.targetpath, selectpn))
+
+ if selectpn:
+ logger.debug('Selecting recipe %s for file %s' % (selectpn, args.targetpath))
+ if postinst_pns:
+ logger.warn('%s be modified by postinstall scripts for the following recipes:\n %s\nThis may or may not be an issue depending on what modifications these postinstall scripts make.' % (args.targetpath, '\n '.join(postinst_pns)))
+ rd = _parse_recipe(selectpn, tinfoil)
+ if not rd:
+ # Error message already shown
+ return 1
+ sourcefile, instelements, modpatches = determine_file_source(args.targetpath, rd)
+ sourcepath = None
+ if sourcefile:
+ sourcetype, sourcepath = sourcefile.split('://', 1)
+ logger.debug('Original source file is %s (%s)' % (sourcepath, sourcetype))
+ if sourcetype == 'patch':
+ logger.warn('File %s is added by the patch %s - you may need to remove or replace this patch in order to replace the file.' % (args.targetpath, sourcepath))
+ sourcepath = None
+ else:
+ logger.debug('Unable to determine source file, proceeding anyway')
+ if modpatches:
+ logger.warn('File %s is modified by the following patches:\n %s' % (args.targetpath, '\n '.join(modpatches)))
+
+ if instelements and sourcepath:
+ install = None
+ else:
+ # Auto-determine permissions
+ # Check destination
+ binpaths = '${bindir}:${sbindir}:${base_bindir}:${base_sbindir}:${libexecdir}:${sysconfdir}/init.d'
+ perms = '0644'
+ if os.path.abspath(os.path.dirname(args.targetpath)) in rd.expand(binpaths).split(':'):
+ # File is going into a directory normally reserved for executables, so it should be executable
+ perms = '0755'
+ else:
+ # Check source
+ st = os.stat(args.newfile)
+ if st.st_mode & stat.S_IXUSR:
+ perms = '0755'
+ install = {args.newfile: (args.targetpath, perms)}
+ oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine)
+ return 0
+ else:
+ if alternative_pns:
+ logger.error('File %s is an alternative possibly provided by the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(alternative_pns)))
+ elif postinst_pns:
+ logger.error('File %s may be written out in a pre/postinstall script of the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(postinst_pns)))
+ return 3
+
+
+def appendsrc(args, files, rd, extralines=None):
+ import oe.recipeutils
+
+ srcdir = rd.getVar('S')
+ workdir = rd.getVar('WORKDIR')
+
+ import bb.fetch
+ simplified = {}
+ src_uri = rd.getVar('SRC_URI').split()
+ for uri in src_uri:
+ if uri.endswith(';'):
+ uri = uri[:-1]
+ simple_uri = bb.fetch.URI(uri)
+ simple_uri.params = {}
+ simplified[str(simple_uri)] = uri
+
+ copyfiles = {}
+ extralines = extralines or []
+ for newfile, srcfile in files.items():
+ src_destdir = os.path.dirname(srcfile)
+ if not args.use_workdir:
+ if rd.getVar('S') == rd.getVar('STAGING_KERNEL_DIR'):
+ srcdir = os.path.join(workdir, 'git')
+ if not bb.data.inherits_class('kernel-yocto', rd):
+ logger.warn('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/git')
+ src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir)
+ src_destdir = os.path.normpath(src_destdir)
+
+ source_uri = 'file://{0}'.format(os.path.basename(srcfile))
+ if src_destdir and src_destdir != '.':
+ source_uri += ';subdir={0}'.format(src_destdir)
+
+ simple = bb.fetch.URI(source_uri)
+ simple.params = {}
+ simple_str = str(simple)
+ if simple_str in simplified:
+ existing = simplified[simple_str]
+ if source_uri != existing:
+ logger.warn('{0!r} is already in SRC_URI, with different parameters: {1!r}, not adding'.format(source_uri, existing))
+ else:
+ logger.warn('{0!r} is already in SRC_URI, not adding'.format(source_uri))
+ else:
+ extralines.append('SRC_URI += {0}'.format(source_uri))
+ copyfiles[newfile] = srcfile
+
+ oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines)
+
+
+def appendsrcfiles(parser, args):
+ recipedata = _parse_recipe(args.recipe, tinfoil)
+ if not recipedata:
+ parser.error('RECIPE must be a valid recipe name')
+
+ files = dict((f, os.path.join(args.destdir, os.path.basename(f)))
+ for f in args.files)
+ return appendsrc(args, files, recipedata)
+
+
+def appendsrcfile(parser, args):
+ recipedata = _parse_recipe(args.recipe, tinfoil)
+ if not recipedata:
+ parser.error('RECIPE must be a valid recipe name')
+
+ if not args.destfile:
+ args.destfile = os.path.basename(args.file)
+ elif args.destfile.endswith('/'):
+ args.destfile = os.path.join(args.destfile, os.path.basename(args.file))
+
+ return appendsrc(args, {args.file: args.destfile}, recipedata)
+
+
+def layer(layerpath):
+ if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')):
+ raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath))
+ return layerpath
+
+
+def existing_path(filepath):
+ if not os.path.exists(filepath):
+ raise argparse.ArgumentTypeError('{0!r} must be an existing path'.format(filepath))
+ return filepath
+
+
+def existing_file(filepath):
+ filepath = existing_path(filepath)
+ if os.path.isdir(filepath):
+ raise argparse.ArgumentTypeError('{0!r} must be a file, not a directory'.format(filepath))
+ return filepath
+
+
+def destination_path(destpath):
+ if os.path.isabs(destpath):
+ raise argparse.ArgumentTypeError('{0!r} must be a relative path, not absolute'.format(destpath))
+ return destpath
+
+
+def target_path(targetpath):
+ if not os.path.isabs(targetpath):
+ raise argparse.ArgumentTypeError('{0!r} must be an absolute path, not relative'.format(targetpath))
+ return targetpath
+
+
+def register_commands(subparsers):
+ common = argparse.ArgumentParser(add_help=False)
+ common.add_argument('-m', '--machine', help='Make bbappend changes specific to a machine only', metavar='MACHINE')
+ common.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true')
+ common.add_argument('destlayer', metavar='DESTLAYER', help='Base directory of the destination layer to write the bbappend to', type=layer)
+
+ parser_appendfile = subparsers.add_parser('appendfile',
+ parents=[common],
+ help='Create/update a bbappend to replace a target file',
+ description='Creates a bbappend (or updates an existing one) to replace the specified file that appears in the target system, determining the recipe that packages the file and the required path and name for the bbappend automatically. Note that the ability to determine the recipe packaging a particular file depends upon the recipe\'s do_packagedata task having already run prior to running this command (which it will have when the recipe has been built successfully, which in turn will have happened if one or more of the recipe\'s packages is included in an image that has been built successfully).')
+ parser_appendfile.add_argument('targetpath', help='Path to the file to be replaced (as it would appear within the target image, e.g. /etc/motd)', type=target_path)
+ parser_appendfile.add_argument('newfile', help='Custom file to replace the target file with', type=existing_file)
+ parser_appendfile.add_argument('-r', '--recipe', help='Override recipe to apply to (default is to find which recipe already packages the file)')
+ parser_appendfile.set_defaults(func=appendfile, parserecipes=True)
+
+ common_src = argparse.ArgumentParser(add_help=False, parents=[common])
+ common_src.add_argument('-W', '--workdir', help='Unpack file into WORKDIR rather than S', dest='use_workdir', action='store_true')
+ common_src.add_argument('recipe', metavar='RECIPE', help='Override recipe to apply to')
+
+ parser = subparsers.add_parser('appendsrcfiles',
+ parents=[common_src],
+ help='Create/update a bbappend to add or replace source files',
+ description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.')
+ parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path)
+ parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path)
+ parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True)
+
+ parser = subparsers.add_parser('appendsrcfile',
+ parents=[common_src],
+ help='Create/update a bbappend to add or replace a source file',
+ description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.')
+ parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path)
+ parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path)
+ parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True)
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 0c413688c0..5af58a12f7 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -1,6 +1,6 @@
# Recipe creation tool - create command plugin
#
-# Copyright (C) 2014-2015 Intel Corporation
+# Copyright (C) 2014-2016 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -21,14 +21,29 @@ import argparse
import glob
import fnmatch
import re
+import json
import logging
import scriptutils
+from urllib.parse import urlparse, urldefrag, urlsplit
+import hashlib
logger = logging.getLogger('recipetool')
tinfoil = None
plugins = None
+def log_error_cond(message, debugonly):
+ if debugonly:
+ logger.debug(message)
+ else:
+ logger.error(message)
+
+def log_info_cond(message, debugonly):
+ if debugonly:
+ logger.debug(message)
+ else:
+ logger.info(message)
+
def plugin_init(pluginlist):
# Take a reference to the list so we can use it later
global plugins
@@ -38,102 +53,465 @@ def tinfoil_init(instance):
global tinfoil
tinfoil = instance
-class RecipeHandler():
+class RecipeHandler(object):
+ recipelibmap = {}
+ recipeheadermap = {}
+ recipecmakefilemap = {}
+ recipebinmap = {}
+
+ def __init__(self):
+ self._devtool = False
+
+ @staticmethod
+ def load_libmap(d):
+ '''Load library->recipe mapping'''
+ import oe.package
+
+ if RecipeHandler.recipelibmap:
+ return
+ # First build up library->package mapping
+ shlib_providers = oe.package.read_shlib_providers(d)
+ libdir = d.getVar('libdir')
+ base_libdir = d.getVar('base_libdir')
+ libpaths = list(set([base_libdir, libdir]))
+ libname_re = re.compile('^lib(.+)\.so.*$')
+ pkglibmap = {}
+ for lib, item in shlib_providers.items():
+ for path, pkg in item.items():
+ if path in libpaths:
+ res = libname_re.match(lib)
+ if res:
+ libname = res.group(1)
+ if not libname in pkglibmap:
+ pkglibmap[libname] = pkg[0]
+ else:
+ logger.debug('unable to extract library name from %s' % lib)
+
+ # Now turn it into a library->recipe mapping
+ pkgdata_dir = d.getVar('PKGDATA_DIR')
+ for libname, pkg in pkglibmap.items():
+ try:
+ with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
+ for line in f:
+ if line.startswith('PN:'):
+ RecipeHandler.recipelibmap[libname] = line.split(':', 1)[-1].strip()
+ break
+ except IOError as ioe:
+ if ioe.errno == 2:
+ logger.warn('unable to find a pkgdata file for package %s' % pkg)
+ else:
+ raise
+
+ # Some overrides - these should be mapped to the virtual
+ RecipeHandler.recipelibmap['GL'] = 'virtual/libgl'
+ RecipeHandler.recipelibmap['EGL'] = 'virtual/egl'
+ RecipeHandler.recipelibmap['GLESv2'] = 'virtual/libgles2'
+
+ @staticmethod
+ def load_devel_filemap(d):
+ '''Build up development file->recipe mapping'''
+ if RecipeHandler.recipeheadermap:
+ return
+ pkgdata_dir = d.getVar('PKGDATA_DIR')
+ includedir = d.getVar('includedir')
+ cmakedir = os.path.join(d.getVar('libdir'), 'cmake')
+ for pkg in glob.glob(os.path.join(pkgdata_dir, 'runtime', '*-dev')):
+ with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
+ pn = None
+ headers = []
+ cmakefiles = []
+ for line in f:
+ if line.startswith('PN:'):
+ pn = line.split(':', 1)[-1].strip()
+ elif line.startswith('FILES_INFO:'):
+ val = line.split(':', 1)[1].strip()
+ dictval = json.loads(val)
+ for fullpth in sorted(dictval):
+ if fullpth.startswith(includedir) and fullpth.endswith('.h'):
+ headers.append(os.path.relpath(fullpth, includedir))
+ elif fullpth.startswith(cmakedir) and fullpth.endswith('.cmake'):
+ cmakefiles.append(os.path.relpath(fullpth, cmakedir))
+ if pn and headers:
+ for header in headers:
+ RecipeHandler.recipeheadermap[header] = pn
+ if pn and cmakefiles:
+ for fn in cmakefiles:
+ RecipeHandler.recipecmakefilemap[fn] = pn
+
@staticmethod
- def checkfiles(path, speclist):
+ def load_binmap(d):
+ '''Build up native binary->recipe mapping'''
+ if RecipeHandler.recipebinmap:
+ return
+ sstate_manifests = d.getVar('SSTATE_MANIFESTS')
+ staging_bindir_native = d.getVar('STAGING_BINDIR_NATIVE')
+ build_arch = d.getVar('BUILD_ARCH')
+ fileprefix = 'manifest-%s-' % build_arch
+ for fn in glob.glob(os.path.join(sstate_manifests, '%s*-native.populate_sysroot' % fileprefix)):
+ with open(fn, 'r') as f:
+ pn = os.path.basename(fn).rsplit('.', 1)[0][len(fileprefix):]
+ for line in f:
+ if line.startswith(staging_bindir_native):
+ prog = os.path.basename(line.rstrip())
+ RecipeHandler.recipebinmap[prog] = pn
+
+ @staticmethod
+ def checkfiles(path, speclist, recursive=False):
results = []
- for spec in speclist:
- results.extend(glob.glob(os.path.join(path, spec)))
+ if recursive:
+ for root, _, files in os.walk(path):
+ for fn in files:
+ for spec in speclist:
+ if fnmatch.fnmatch(fn, spec):
+ results.append(os.path.join(root, fn))
+ else:
+ for spec in speclist:
+ results.extend(glob.glob(os.path.join(path, spec)))
return results
- def genfunction(self, outlines, funcname, content):
- outlines.append('%s () {' % funcname)
+ @staticmethod
+ def handle_depends(libdeps, pcdeps, deps, outlines, values, d):
+ if pcdeps:
+ recipemap = read_pkgconfig_provides(d)
+ if libdeps:
+ RecipeHandler.load_libmap(d)
+
+ ignorelibs = ['socket']
+ ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native', 'coreutils-native']
+
+ unmappedpc = []
+ pcdeps = list(set(pcdeps))
+ for pcdep in pcdeps:
+ if isinstance(pcdep, str):
+ recipe = recipemap.get(pcdep, None)
+ if recipe:
+ deps.append(recipe)
+ else:
+ if not pcdep.startswith('$'):
+ unmappedpc.append(pcdep)
+ else:
+ for item in pcdep:
+ recipe = recipemap.get(pcdep, None)
+ if recipe:
+ deps.append(recipe)
+ break
+ else:
+ unmappedpc.append('(%s)' % ' or '.join(pcdep))
+
+ unmappedlibs = []
+ for libdep in libdeps:
+ if isinstance(libdep, tuple):
+ lib, header = libdep
+ else:
+ lib = libdep
+ header = None
+
+ if lib in ignorelibs:
+ logger.debug('Ignoring library dependency %s' % lib)
+ continue
+
+ recipe = RecipeHandler.recipelibmap.get(lib, None)
+ if recipe:
+ deps.append(recipe)
+ elif recipe is None:
+ if header:
+ RecipeHandler.load_devel_filemap(d)
+ recipe = RecipeHandler.recipeheadermap.get(header, None)
+ if recipe:
+ deps.append(recipe)
+ elif recipe is None:
+ unmappedlibs.append(lib)
+ else:
+ unmappedlibs.append(lib)
+
+ deps = set(deps).difference(set(ignoredeps))
+
+ if unmappedpc:
+ outlines.append('# NOTE: unable to map the following pkg-config dependencies: %s' % ' '.join(unmappedpc))
+ outlines.append('# (this is based on recipes that have previously been built and packaged)')
+
+ if unmappedlibs:
+ outlines.append('# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmappedlibs))))
+ outlines.append('# (this is based on recipes that have previously been built and packaged)')
+
+ if deps:
+ values['DEPENDS'] = ' '.join(deps)
+
+ @staticmethod
+ def genfunction(outlines, funcname, content, python=False, forcespace=False):
+ if python:
+ prefix = 'python '
+ else:
+ prefix = ''
+ outlines.append('%s%s () {' % (prefix, funcname))
+ if python or forcespace:
+ indent = ' '
+ else:
+ indent = '\t'
+ addnoop = not python
for line in content:
- outlines.append('\t%s' % line)
+ outlines.append('%s%s' % (indent, line))
+ if addnoop:
+ strippedline = line.lstrip()
+ if strippedline and not strippedline.startswith('#'):
+ addnoop = False
+ if addnoop:
+ # Without this there'll be a syntax error
+ outlines.append('%s:' % indent)
outlines.append('}')
outlines.append('')
- def process(self, srctree, classes, lines_before, lines_after, handled):
+ def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
return False
+def validate_pv(pv):
+ if not pv or '_version' in pv.lower() or pv[0] not in '0123456789':
+ return False
+ return True
+
+def determine_from_filename(srcfile):
+ """Determine name and version from a filename"""
+ if is_package(srcfile):
+ # Force getting the value from the package metadata
+ return None, None
-def fetch_source(uri, destdir):
- import bb.data
- bb.utils.mkdirhier(destdir)
+ if '.tar.' in srcfile:
+ namepart = srcfile.split('.tar.')[0]
+ else:
+ namepart = os.path.splitext(srcfile)[0]
+ namepart = namepart.lower().replace('_', '-')
+ if namepart.endswith('.src'):
+ namepart = namepart[:-4]
+ if namepart.endswith('.orig'):
+ namepart = namepart[:-5]
+ splitval = namepart.split('-')
+ logger.debug('determine_from_filename: split name %s into: %s' % (srcfile, splitval))
+
+ ver_re = re.compile('^v?[0-9]')
+
+ pv = None
+ pn = None
+ if len(splitval) == 1:
+ # Try to split the version out if there is no separator (or a .)
+ res = re.match('^([^0-9]+)([0-9.]+.*)$', namepart)
+ if res:
+ if len(res.group(1)) > 1 and len(res.group(2)) > 1:
+ pn = res.group(1).rstrip('.')
+ pv = res.group(2)
+ else:
+ pn = namepart
+ else:
+ if splitval[-1] in ['source', 'src']:
+ splitval.pop()
+ if len(splitval) > 2 and re.match('^(alpha|beta|stable|release|rc[0-9]|pre[0-9]|p[0-9]|[0-9]{8})', splitval[-1]) and ver_re.match(splitval[-2]):
+ pv = '-'.join(splitval[-2:])
+ if pv.endswith('-release'):
+ pv = pv[:-8]
+ splitval = splitval[:-2]
+ elif ver_re.match(splitval[-1]):
+ pv = splitval.pop()
+ pn = '-'.join(splitval)
+ if pv and pv.startswith('v'):
+ pv = pv[1:]
+ logger.debug('determine_from_filename: name = "%s" version = "%s"' % (pn, pv))
+ return (pn, pv)
+
+def determine_from_url(srcuri):
+ """Determine name and version from a URL"""
+ pn = None
+ pv = None
+ parseres = urlparse(srcuri.lower().split(';', 1)[0])
+ if parseres.path:
+ if 'github.com' in parseres.netloc:
+ res = re.search(r'.*/(.*?)/archive/(.*)-final\.(tar|zip)', parseres.path)
+ if res:
+ pn = res.group(1).strip().replace('_', '-')
+ pv = res.group(2).strip().replace('_', '.')
+ else:
+ res = re.search(r'.*/(.*?)/archive/v?(.*)\.(tar|zip)', parseres.path)
+ if res:
+ pn = res.group(1).strip().replace('_', '-')
+ pv = res.group(2).strip().replace('_', '.')
+ elif 'bitbucket.org' in parseres.netloc:
+ res = re.search(r'.*/(.*?)/get/[a-zA-Z_-]*([0-9][0-9a-zA-Z_.]*)\.(tar|zip)', parseres.path)
+ if res:
+ pn = res.group(1).strip().replace('_', '-')
+ pv = res.group(2).strip().replace('_', '.')
+
+ if not pn and not pv and parseres.scheme not in ['git', 'gitsm', 'svn', 'hg']:
+ srcfile = os.path.basename(parseres.path.rstrip('/'))
+ pn, pv = determine_from_filename(srcfile)
+
+ logger.debug('Determined from source URL: name = "%s", version = "%s"' % (pn, pv))
+ return (pn, pv)
+
+def supports_srcrev(uri):
localdata = bb.data.createCopy(tinfoil.config_data)
- bb.data.update_data(localdata)
- localdata.setVar('BB_STRICT_CHECKSUM', '')
+ # This is a bit sad, but if you don't have this set there can be some
+ # odd interactions with the urldata cache which lead to errors
localdata.setVar('SRCREV', '${AUTOREV}')
- ret = (None, None)
- olddir = os.getcwd()
try:
fetcher = bb.fetch2.Fetch([uri], localdata)
- for u in fetcher.ud:
- ud = fetcher.ud[u]
- ud.ignore_checksums = True
- fetcher.download()
- fetcher.unpack(destdir)
- for u in fetcher.ud:
- ud = fetcher.ud[u]
- if ud.method.recommends_checksum(ud):
- md5value = bb.utils.md5_file(ud.localpath)
- sha256value = bb.utils.sha256_file(ud.localpath)
- ret = (md5value, sha256value)
- except bb.fetch2.BBFetchException, e:
- raise bb.build.FuncFailed(e)
- finally:
- os.chdir(olddir)
- return ret
-
-def supports_srcrev(uri):
- localdata = bb.data.createCopy(tinfoil.config_data)
- bb.data.update_data(localdata)
- fetcher = bb.fetch2.Fetch([uri], localdata)
- urldata = fetcher.ud
- for u in urldata:
- if urldata[u].method.supports_srcrev():
+ urldata = fetcher.ud
+ for u in urldata:
+ if urldata[u].method.supports_srcrev():
+ return True
+ except bb.fetch2.FetchError as e:
+ logger.debug('FetchError in supports_srcrev: %s' % str(e))
+ # Fall back to basic check
+ if uri.startswith(('git://', 'gitsm://')):
return True
return False
+def reformat_git_uri(uri):
+ '''Convert any http[s]://....git URI into git://...;protocol=http[s]'''
+ checkuri = uri.split(';', 1)[0]
+ if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri):
+ res = re.match('(http|https|ssh)://([^;]+(\.git)?)(;.*)?$', uri)
+ if res:
+ # Need to switch the URI around so that the git fetcher is used
+ return 'git://%s;protocol=%s%s' % (res.group(2), res.group(1), res.group(4) or '')
+ elif '@' in checkuri:
+ # Catch e.g. git@git.example.com:repo.git
+ return 'git://%s;protocol=ssh' % checkuri.replace(':', '/', 1)
+ return uri
+
+def is_package(url):
+ '''Check if a URL points to a package'''
+ checkurl = url.split(';', 1)[0]
+ if checkurl.endswith(('.deb', '.ipk', '.rpm', '.srpm')):
+ return True
+ return False
+
def create_recipe(args):
import bb.process
import tempfile
import shutil
+ import oe.recipeutils
pkgarch = ""
if args.machine:
pkgarch = "${MACHINE_ARCH}"
+ extravalues = {}
checksums = (None, None)
tempsrc = ''
+ source = args.source
srcsubdir = ''
- if '://' in args.source:
+ srcrev = '${AUTOREV}'
+
+ if os.path.isfile(source):
+ source = 'file://%s' % os.path.abspath(source)
+
+ if scriptutils.is_src_url(source):
# Fetch a URL
- srcuri = args.source
+ fetchuri = reformat_git_uri(urldefrag(source)[0])
+ if args.binary:
+ # Assume the archive contains the directory structure verbatim
+ # so we need to extract to a subdirectory
+ fetchuri += ';subdir=${BP}'
+ srcuri = fetchuri
+ rev_re = re.compile(';rev=([^;]+)')
+ res = rev_re.search(srcuri)
+ if res:
+ srcrev = res.group(1)
+ srcuri = rev_re.sub('', srcuri)
tempsrc = tempfile.mkdtemp(prefix='recipetool-')
srctree = tempsrc
+ d = bb.data.createCopy(tinfoil.config_data)
+ if fetchuri.startswith('npm://'):
+ # Check if npm is available
+ npm_bindir = check_npm(tinfoil, args.devtool)
+ d.prependVar('PATH', '%s:' % npm_bindir)
logger.info('Fetching %s...' % srcuri)
- checksums = fetch_source(args.source, srctree)
+ try:
+ checksums = scriptutils.fetch_uri(d, fetchuri, srctree, srcrev)
+ except bb.fetch2.BBFetchException as e:
+ logger.error(str(e).rstrip())
+ sys.exit(1)
dirlist = os.listdir(srctree)
if 'git.indirectionsymlink' in dirlist:
dirlist.remove('git.indirectionsymlink')
- if len(dirlist) == 1 and os.path.isdir(os.path.join(srctree, dirlist[0])):
- # We unpacked a single directory, so we should use that
- srcsubdir = dirlist[0]
- srctree = os.path.join(srctree, srcsubdir)
+ if len(dirlist) == 1:
+ singleitem = os.path.join(srctree, dirlist[0])
+ if os.path.isdir(singleitem):
+ # We unpacked a single directory, so we should use that
+ srcsubdir = dirlist[0]
+ srctree = os.path.join(srctree, srcsubdir)
+ else:
+ with open(singleitem, 'r', errors='surrogateescape') as f:
+ if '<html' in f.read(100).lower():
+ logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri)
+ sys.exit(1)
+ if os.path.exists(os.path.join(srctree, '.gitmodules')) and srcuri.startswith('git://'):
+ srcuri = 'gitsm://' + srcuri[6:]
+ logger.info('Fetching submodules...')
+ bb.process.run('git submodule update --init --recursive', cwd=srctree)
+
+ if is_package(fetchuri):
+ tmpfdir = tempfile.mkdtemp(prefix='recipetool-')
+ try:
+ pkgfile = None
+ try:
+ fileuri = fetchuri + ';unpack=0'
+ scriptutils.fetch_uri(tinfoil.config_data, fileuri, tmpfdir, srcrev)
+ for root, _, files in os.walk(tmpfdir):
+ for f in files:
+ pkgfile = os.path.join(root, f)
+ break
+ except bb.fetch2.BBFetchException as e:
+ logger.warn('Second fetch to get metadata failed: %s' % str(e).rstrip())
+
+ if pkgfile:
+ if pkgfile.endswith(('.deb', '.ipk')):
+ stdout, _ = bb.process.run('ar x %s' % pkgfile, cwd=tmpfdir)
+ stdout, _ = bb.process.run('tar xf control.tar.gz', cwd=tmpfdir)
+ values = convert_debian(tmpfdir)
+ extravalues.update(values)
+ elif pkgfile.endswith(('.rpm', '.srpm')):
+ stdout, _ = bb.process.run('rpm -qp --xml %s > pkginfo.xml' % pkgfile, cwd=tmpfdir)
+ values = convert_rpm_xml(os.path.join(tmpfdir, 'pkginfo.xml'))
+ extravalues.update(values)
+ finally:
+ shutil.rmtree(tmpfdir)
else:
# Assume we're pointing to an existing source tree
if args.extract_to:
logger.error('--extract-to cannot be specified if source is a directory')
sys.exit(1)
- if not os.path.isdir(args.source):
- logger.error('Invalid source directory %s' % args.source)
+ if not os.path.isdir(source):
+ logger.error('Invalid source directory %s' % source)
sys.exit(1)
+ srctree = source
srcuri = ''
- srctree = args.source
+ if os.path.exists(os.path.join(srctree, '.git')):
+ # Try to get upstream repo location from origin remote
+ try:
+ stdout, _ = bb.process.run('git remote -v', cwd=srctree, shell=True)
+ except bb.process.ExecutionError as e:
+ stdout = None
+ if stdout:
+ for line in stdout.splitlines():
+ splitline = line.split()
+ if len(splitline) > 1:
+ if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]):
+ srcuri = reformat_git_uri(splitline[1])
+ srcsubdir = 'git'
+ break
- outfile = args.outfile
+ if args.src_subdir:
+ srcsubdir = os.path.join(srcsubdir, args.src_subdir)
+ srctree_use = os.path.join(srctree, args.src_subdir)
+ else:
+ srctree_use = srctree
+
+ if args.outfile and os.path.isdir(args.outfile):
+ outfile = None
+ outdir = args.outfile
+ else:
+ outfile = args.outfile
+ outdir = None
if outfile and outfile != '-':
if os.path.exists(outfile):
logger.error('Output file %s already exists' % outfile)
@@ -145,52 +523,55 @@ def create_recipe(args):
lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0]))
lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.')
lines_before.append('# (Feel free to remove these comments when editing.)')
- lines_before.append('#')
-
- licvalues = guess_license(srctree)
- lic_files_chksum = []
- if licvalues:
- licenses = []
- for licvalue in licvalues:
- if not licvalue[0] in licenses:
- licenses.append(licvalue[0])
- lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
- lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is')
- lines_before.append('# your responsibility to verify that the values are complete and correct.')
- if len(licvalues) > 1:
- lines_before.append('#')
- lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate')
- lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there')
- lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying')
- lines_before.append('# documentation to determine which situation is applicable.')
- else:
- lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying')
- lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.')
- lines_before.append('#')
- lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if')
- lines_before.append('# this is not accurate with respect to the licensing of the software being built (it')
- lines_before.append('# will not be in most cases) you must specify the correct value before using this')
- lines_before.append('# recipe for anything other than initial testing/development!')
- licenses = ['CLOSED']
- lines_before.append('LICENSE = "%s"' % ' '.join(licenses))
- lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum))
+ # We need a blank line here so that patch_recipe_lines can rewind before the LICENSE comments
lines_before.append('')
+ handled = []
+ licvalues = handle_license_vars(srctree_use, lines_before, handled, extravalues, tinfoil.config_data)
+
+ classes = []
+
# FIXME This is kind of a hack, we probably ought to be using bitbake to do this
- # we'd also want a way to automatically set outfile based upon auto-detecting these values from the source if possible
- recipefn = os.path.splitext(os.path.basename(outfile))[0]
- fnsplit = recipefn.split('_')
- if len(fnsplit) > 1:
- pn = fnsplit[0]
- pv = fnsplit[1]
- else:
- pn = recipefn
- pv = None
+ pn = None
+ pv = None
+ if outfile:
+ recipefn = os.path.splitext(os.path.basename(outfile))[0]
+ fnsplit = recipefn.split('_')
+ if len(fnsplit) > 1:
+ pn = fnsplit[0]
+ pv = fnsplit[1]
+ else:
+ pn = recipefn
- if srcuri:
- if pv and pv not in 'git svn hg'.split():
- srcuri = srcuri.replace(pv, '${PV}')
+ if args.version:
+ pv = args.version
+
+ if args.name:
+ pn = args.name
+ if args.name.endswith('-native'):
+ if args.also_native:
+ logger.error('--also-native cannot be specified for a recipe named *-native (*-native denotes a recipe that is already only for native) - either remove the -native suffix from the name or drop --also-native')
+ sys.exit(1)
+ classes.append('native')
+ elif args.name.startswith('nativesdk-'):
+ if args.also_native:
+ logger.error('--also-native cannot be specified for a recipe named nativesdk-* (nativesdk-* denotes a recipe that is already only for nativesdk)')
+ sys.exit(1)
+ classes.append('nativesdk')
+
+ if pv and pv not in 'git svn hg'.split():
+ realpv = pv
else:
+ realpv = None
+
+ if srcuri and not realpv or not pn:
+ name_pn, name_pv = determine_from_url(srcuri)
+ if name_pn and not pn:
+ pn = name_pn
+ if name_pv and not realpv:
+ realpv = name_pv
+
+ if not srcuri:
lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
lines_before.append('SRC_URI = "%s"' % srcuri)
(md5value, sha256value) = checksums
@@ -201,17 +582,17 @@ def create_recipe(args):
if srcuri and supports_srcrev(srcuri):
lines_before.append('')
lines_before.append('# Modify these as desired')
- lines_before.append('PV = "1.0+git${SRCPV}"')
- lines_before.append('SRCREV = "${AUTOREV}"')
+ lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0'))
+ if not args.autorev and srcrev == '${AUTOREV}':
+ if os.path.exists(os.path.join(srctree, '.git')):
+ (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
+ srcrev = stdout.rstrip()
+ lines_before.append('SRCREV = "%s"' % srcrev)
lines_before.append('')
- if srcsubdir and pv:
- if srcsubdir == "%s-%s" % (pn, pv):
- # This would be the default, so we don't need to set S in the recipe
- srcsubdir = ''
- if srcsubdir:
- if pv and pv not in 'git svn hg'.split():
- srcsubdir = srcsubdir.replace(pv, '${PV}')
+ if srcsubdir and not args.binary:
+ # (for binary packages we explicitly specify subdir= when fetching to
+ # match the default value of S, so we don't need to set it in that case)
lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
lines_before.append('')
@@ -219,48 +600,253 @@ def create_recipe(args):
lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch)
lines_after.append('')
+ if args.binary:
+ lines_after.append('INSANE_SKIP_${PN} += "already-stripped"')
+ lines_after.append('')
+
+ if args.fetch_dev:
+ extravalues['fetchdev'] = True
+ else:
+ extravalues['fetchdev'] = None
+
# Find all plugins that want to register handlers
- handlers = []
+ logger.debug('Loading recipe handlers')
+ raw_handlers = []
for plugin in plugins:
if hasattr(plugin, 'register_recipe_handlers'):
- plugin.register_recipe_handlers(handlers)
+ plugin.register_recipe_handlers(raw_handlers)
+ # Sort handlers by priority
+ handlers = []
+ for i, handler in enumerate(raw_handlers):
+ if isinstance(handler, tuple):
+ handlers.append((handler[0], handler[1], i))
+ else:
+ handlers.append((handler, 0, i))
+ handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
+ for handler, priority, _ in handlers:
+ logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
+ setattr(handler, '_devtool', args.devtool)
+ handlers = [item[0] for item in handlers]
# Apply the handlers
- classes = []
- handled = []
+ if args.binary:
+ classes.append('bin_package')
+ handled.append('buildsystem')
+
for handler in handlers:
- handler.process(srctree, classes, lines_before, lines_after, handled)
+ handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
+
+ extrafiles = extravalues.pop('extrafiles', {})
+ extra_pn = extravalues.pop('PN', None)
+ extra_pv = extravalues.pop('PV', None)
+
+ if extra_pv and not realpv:
+ realpv = extra_pv
+ if not validate_pv(realpv):
+ realpv = None
+ else:
+ realpv = realpv.lower().split()[0]
+ if '_' in realpv:
+ realpv = realpv.replace('_', '-')
+ if extra_pn and not pn:
+ pn = extra_pn
+ if pn.startswith('GNU '):
+ pn = pn[4:]
+ if ' ' in pn:
+ # Probably a descriptive identifier rather than a proper name
+ pn = None
+ else:
+ pn = pn.lower()
+ if '_' in pn:
+ pn = pn.replace('_', '-')
+
+ if not outfile:
+ if not pn:
+ log_error_cond('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile', args.devtool)
+ # devtool looks for this specific exit code, so don't change it
+ sys.exit(15)
+ else:
+ if srcuri and srcuri.startswith(('gitsm://', 'git://', 'hg://', 'svn://')):
+ suffix = srcuri.split(':', 1)[0]
+ if suffix == 'gitsm':
+ suffix = 'git'
+ outfile = '%s_%s.bb' % (pn, suffix)
+ elif realpv:
+ outfile = '%s_%s.bb' % (pn, realpv)
+ else:
+ outfile = '%s.bb' % pn
+ if outdir:
+ outfile = os.path.join(outdir, outfile)
+ # We need to check this again
+ if os.path.exists(outfile):
+ logger.error('Output file %s already exists' % outfile)
+ sys.exit(1)
+
+ # Move any extra files the plugins created to a directory next to the recipe
+ if extrafiles:
+ if outfile == '-':
+ extraoutdir = pn
+ else:
+ extraoutdir = os.path.join(os.path.dirname(outfile), pn)
+ bb.utils.mkdirhier(extraoutdir)
+ for destfn, extrafile in extrafiles.items():
+ shutil.move(extrafile, os.path.join(extraoutdir, destfn))
+
+ lines = lines_before
+ lines_before = []
+ skipblank = True
+ for line in lines:
+ if skipblank:
+ skipblank = False
+ if not line:
+ continue
+ if line.startswith('S = '):
+ if realpv and pv not in 'git svn hg'.split():
+ line = line.replace(realpv, '${PV}')
+ if pn:
+ line = line.replace(pn, '${BPN}')
+ if line == 'S = "${WORKDIR}/${BPN}-${PV}"':
+ skipblank = True
+ continue
+ elif line.startswith('SRC_URI = '):
+ if realpv:
+ line = line.replace(realpv, '${PV}')
+ elif line.startswith('PV = '):
+ if realpv:
+ line = re.sub('"[^+]*\+', '"%s+' % realpv, line)
+ lines_before.append(line)
+
+ if args.also_native:
+ lines = lines_after
+ lines_after = []
+ bbclassextend = None
+ for line in lines:
+ if line.startswith('BBCLASSEXTEND ='):
+ splitval = line.split('"')
+ if len(splitval) > 1:
+ bbclassextend = splitval[1].split()
+ if not 'native' in bbclassextend:
+ bbclassextend.insert(0, 'native')
+ line = 'BBCLASSEXTEND = "%s"' % ' '.join(bbclassextend)
+ lines_after.append(line)
+ if not bbclassextend:
+ lines_after.append('BBCLASSEXTEND = "native"')
+
+ postinst = ("postinst", extravalues.pop('postinst', None))
+ postrm = ("postrm", extravalues.pop('postrm', None))
+ preinst = ("preinst", extravalues.pop('preinst', None))
+ prerm = ("prerm", extravalues.pop('prerm', None))
+ funcs = [postinst, postrm, preinst, prerm]
+ for func in funcs:
+ if func[1]:
+ RecipeHandler.genfunction(lines_after, 'pkg_%s_${PN}' % func[0], func[1])
outlines = []
outlines.extend(lines_before)
if classes:
+ if outlines[-1] and not outlines[-1].startswith('#'):
+ outlines.append('')
outlines.append('inherit %s' % ' '.join(classes))
outlines.append('')
outlines.extend(lines_after)
+ if extravalues:
+ if 'LICENSE' in extravalues and not licvalues:
+ # Don't blow away 'CLOSED' value that comments say we set
+ del extravalues['LICENSE']
+ _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False)
+
if args.extract_to:
scriptutils.git_convert_standalone_clone(srctree)
+ if os.path.isdir(args.extract_to):
+ # If the directory exists we'll move the temp dir into it instead of
+ # its contents - of course, we could try to always move its contents
+ # but that is a pain if there are symlinks; the simplest solution is
+ # to just remove it first
+ os.rmdir(args.extract_to)
shutil.move(srctree, args.extract_to)
- logger.info('Source extracted to %s' % args.extract_to)
+ if tempsrc == srctree:
+ tempsrc = None
+ log_info_cond('Source extracted to %s' % args.extract_to, args.devtool)
if outfile == '-':
sys.stdout.write('\n'.join(outlines) + '\n')
else:
with open(outfile, 'w') as f:
- f.write('\n'.join(outlines) + '\n')
- logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile)
+ lastline = None
+ for line in outlines:
+ if not lastline and not line:
+ # Skip extra blank lines
+ continue
+ f.write('%s\n' % line)
+ lastline = line
+ log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool)
if tempsrc:
- shutil.rmtree(tempsrc)
+ if args.keep_temp:
+ logger.info('Preserving temporary directory %s' % tempsrc)
+ else:
+ shutil.rmtree(tempsrc)
return 0
+def handle_license_vars(srctree, lines_before, handled, extravalues, d):
+ licvalues = guess_license(srctree, d)
+ lic_files_chksum = []
+ lic_unknown = []
+ if licvalues:
+ licenses = []
+ for licvalue in licvalues:
+ if not licvalue[0] in licenses:
+ licenses.append(licvalue[0])
+ lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
+ if licvalue[0] == 'Unknown':
+ lic_unknown.append(licvalue[1])
+ lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is')
+ lines_before.append('# your responsibility to verify that the values are complete and correct.')
+ if len(licvalues) > 1:
+ lines_before.append('#')
+ lines_before.append('# NOTE: multiple licenses have been detected; they have been separated with &')
+ lines_before.append('# in the LICENSE value for now since it is a reasonable assumption that all')
+ lines_before.append('# of the licenses apply. If instead there is a choice between the multiple')
+ lines_before.append('# licenses then you should change the value to separate the licenses with |')
+ lines_before.append('# instead of &. If there is any doubt, check the accompanying documentation')
+ lines_before.append('# to determine which situation is applicable.')
+ if lic_unknown:
+ lines_before.append('#')
+ lines_before.append('# The following license files were not able to be identified and are')
+ lines_before.append('# represented as "Unknown" below, you will need to check them yourself:')
+ for licfile in lic_unknown:
+ lines_before.append('# %s' % licfile)
+ lines_before.append('#')
+ else:
+ lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying')
+ lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.')
+ lines_before.append('#')
+ lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if')
+ lines_before.append('# this is not accurate with respect to the licensing of the software being built (it')
+ lines_before.append('# will not be in most cases) you must specify the correct value before using this')
+ lines_before.append('# recipe for anything other than initial testing/development!')
+ licenses = ['CLOSED']
+ pkg_license = extravalues.pop('LICENSE', None)
+ if pkg_license:
+ if licenses == ['Unknown']:
+ lines_before.append('# NOTE: The following LICENSE value was determined from the original package metadata')
+ licenses = [pkg_license]
+ else:
+ lines_before.append('# NOTE: Original package metadata indicates license is: %s' % pkg_license)
+ lines_before.append('LICENSE = "%s"' % ' & '.join(licenses))
+ lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum))
+ lines_before.append('')
+ handled.append(('license', licvalues))
+ return licvalues
+
def get_license_md5sums(d, static_only=False):
import bb.utils
md5sums = {}
if not static_only:
# Gather md5sums of license files in common license dir
- commonlicdir = d.getVar('COMMON_LICENSE_DIR', True)
+ commonlicdir = d.getVar('COMMON_LICENSE_DIR')
for fn in os.listdir(commonlicdir):
md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn))
md5sums[md5value] = fn
@@ -295,19 +881,92 @@ def get_license_md5sums(d, static_only=False):
md5sums['5f30f0716dfdd0d91eb439ebec522ec2'] = 'LGPLv2'
md5sums['55ca817ccb7d5b5b66355690e9abc605'] = 'LGPLv2'
md5sums['252890d9eee26aab7b432e8b8a616475'] = 'LGPLv2'
+ md5sums['3214f080875748938ba060314b4f727d'] = 'LGPLv2'
+ md5sums['db979804f025cf55aabec7129cb671ed'] = 'LGPLv2'
md5sums['d32239bcb673463ab874e80d47fae504'] = 'GPLv3'
md5sums['f27defe1e96c2e1ecd4e0c9be8967949'] = 'GPLv3'
md5sums['6a6a8e020838b23406c81b19c1d46df6'] = 'LGPLv3'
md5sums['3b83ef96387f14655fc854ddc3c6bd57'] = 'Apache-2.0'
md5sums['385c55653886acac3821999a3ccd17b3'] = 'Artistic-1.0 | GPL-2.0' # some perl modules
+ md5sums['54c7042be62e169199200bc6477f04d1'] = 'BSD-3-Clause'
return md5sums
-def guess_license(srctree):
+def crunch_license(licfile):
+ '''
+ Remove non-material text from a license file and then check
+ its md5sum against a known list. This works well for licenses
+ which contain a copyright statement, but is also a useful way
+ to handle people's insistence upon reformatting the license text
+ slightly (with no material difference to the text of the
+ license).
+ '''
+
+ import oe.utils
+
+ # Note: these are carefully constructed!
+ license_title_re = re.compile('^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$')
+ license_statement_re = re.compile('^(This (project|software) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$')
+ copyright_re = re.compile('^(#+)? *Copyright .*$')
+
+ crunched_md5sums = {}
+ # The following two were gleaned from the "forever" npm package
+ crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC'
+ crunched_md5sums['eecf6429523cbc9693547cf2db790b5c'] = 'MIT'
+ # https://github.com/vasi/pixz/blob/master/LICENSE
+ crunched_md5sums['2f03392b40bbe663597b5bd3cc5ebdb9'] = 'BSD-2-Clause'
+ # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt
+ crunched_md5sums['e72e5dfef0b1a4ca8a3d26a60587db66'] = 'BSD-2-Clause'
+ # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE
+ crunched_md5sums['8be76ac6d191671f347ee4916baa637e'] = 'GPLv2'
+ # https://github.com/datto/dattobd/blob/master/COPYING
+ # http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/GPLv2.TXT
+ crunched_md5sums['1d65c5ad4bf6489f85f4812bf08ae73d'] = 'GPLv2'
+ # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt
+ # http://git.neil.brown.name/?p=mdadm.git;a=blob;f=COPYING;h=d159169d1050894d3ea3b98e1c965c4058208fe1;hb=HEAD
+ crunched_md5sums['fb530f66a7a89ce920f0e912b5b66d4b'] = 'GPLv2'
+ # https://github.com/gkos/nrf24/blob/master/COPYING
+ crunched_md5sums['7b6aaa4daeafdfa6ed5443fd2684581b'] = 'GPLv2'
+ # https://github.com/josch09/resetusb/blob/master/COPYING
+ crunched_md5sums['8b8ac1d631a4d220342e83bcf1a1fbc3'] = 'GPLv3'
+ # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1
+ crunched_md5sums['2ea316ed973ae176e502e2297b574bb3'] = 'LGPLv2.1'
+ # unixODBC-2.3.4 COPYING
+ crunched_md5sums['1daebd9491d1e8426900b4fa5a422814'] = 'LGPLv2.1'
+ # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3
+ crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3'
+ lictext = []
+ with open(licfile, 'r', errors='surrogateescape') as f:
+ for line in f:
+ # Drop opening statements
+ if copyright_re.match(line):
+ continue
+ elif license_title_re.match(line):
+ continue
+ elif license_statement_re.match(line):
+ continue
+ # Squash spaces, and replace smart quotes, double quotes
+ # and backticks with single quotes
+ line = oe.utils.squashspaces(line.strip())
+ line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'')
+ if line:
+ lictext.append(line)
+
+ m = hashlib.md5()
+ try:
+ m.update(' '.join(lictext).encode('utf-8'))
+ md5val = m.hexdigest()
+ except UnicodeEncodeError:
+ md5val = None
+ lictext = ''
+ license = crunched_md5sums.get(md5val, None)
+ return license, md5val, lictext
+
+def guess_license(srctree, d):
import bb
- md5sums = get_license_md5sums(tinfoil.config_data)
+ md5sums = get_license_md5sums(d)
licenses = []
- licspecs = ['LICENSE*', 'COPYING*', '*[Ll]icense*', 'LICENCE*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*']
+ licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*']
licfiles = []
for root, dirs, files in os.walk(srctree):
for fn in files:
@@ -318,22 +977,53 @@ def guess_license(srctree):
licfiles.append(fullpath)
for licfile in licfiles:
md5value = bb.utils.md5_file(licfile)
- license = md5sums.get(md5value, 'Unknown')
+ license = md5sums.get(md5value, None)
+ if not license:
+ license, crunched_md5, lictext = crunch_license(licfile)
+ if not license:
+ license = 'Unknown'
licenses.append((license, os.path.relpath(licfile, srctree), md5value))
# FIXME should we grab at least one source file with a license header and add that too?
return licenses
+def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn='${PN}'):
+ """
+ Given a list of (license, path, md5sum) as returned by guess_license(),
+ a dict of package name to path mappings, write out a set of
+ package-specific LICENSE values.
+ """
+ pkglicenses = {pn: []}
+ for license, licpath, _ in licvalues:
+ for pkgname, pkgpath in packages.items():
+ if licpath.startswith(pkgpath + '/'):
+ if pkgname in pkglicenses:
+ pkglicenses[pkgname].append(license)
+ else:
+ pkglicenses[pkgname] = [license]
+ break
+ else:
+ # Accumulate on the main package
+ pkglicenses[pn].append(license)
+ outlicenses = {}
+ for pkgname in packages:
+ license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown'
+ if license == 'Unknown' and pkgname in fallback_licenses:
+ license = fallback_licenses[pkgname]
+ outlines.append('LICENSE_%s = "%s"' % (pkgname, license))
+ outlicenses[pkgname] = license.split()
+ return outlicenses
+
def read_pkgconfig_provides(d):
- pkgdatadir = d.getVar('PKGDATA_DIR', True)
+ pkgdatadir = d.getVar('PKGDATA_DIR')
pkgmap = {}
for fn in glob.glob(os.path.join(pkgdatadir, 'shlibs2', '*.pclist')):
with open(fn, 'r') as f:
for line in f:
pkgmap[os.path.basename(line.rstrip())] = os.path.splitext(os.path.basename(fn))[0]
recipemap = {}
- for pc, pkg in pkgmap.iteritems():
+ for pc, pkg in pkgmap.items():
pkgdatafile = os.path.join(pkgdatadir, 'runtime', pkg)
if os.path.exists(pkgdatafile):
with open(pkgdatafile, 'r') as f:
@@ -342,81 +1032,134 @@ def read_pkgconfig_provides(d):
recipemap[pc] = line.split(':', 1)[1].strip()
return recipemap
-def convert_pkginfo(pkginfofile):
- values = {}
- with open(pkginfofile, 'r') as f:
- indesc = False
- for line in f:
- if indesc:
- if line.strip():
- values['DESCRIPTION'] += ' ' + line.strip()
- else:
- indesc = False
- else:
- splitline = line.split(': ', 1)
- key = line[0]
- value = line[1]
- if key == 'LICENSE':
- for dep in value.split(','):
- dep = dep.split()[0]
- mapped = depmap.get(dep, '')
- if mapped:
- depends.append(mapped)
- elif key == 'License':
- values['LICENSE'] = value
- elif key == 'Summary':
- values['SUMMARY'] = value
- elif key == 'Description':
- values['DESCRIPTION'] = value
- indesc = True
- return values
-
def convert_debian(debpath):
+ value_map = {'Package': 'PN',
+ 'Version': 'PV',
+ 'Section': 'SECTION',
+ 'License': 'LICENSE',
+ 'Homepage': 'HOMEPAGE'}
+
# FIXME extend this mapping - perhaps use distro_alias.inc?
depmap = {'libz-dev': 'zlib'}
values = {}
depends = []
- with open(os.path.join(debpath, 'control')) as f:
+ with open(os.path.join(debpath, 'control'), 'r', errors='surrogateescape') as f:
indesc = False
for line in f:
if indesc:
- if line.strip():
+ if line.startswith(' '):
if line.startswith(' This package contains'):
indesc = False
else:
- values['DESCRIPTION'] += ' ' + line.strip()
+ if 'DESCRIPTION' in values:
+ values['DESCRIPTION'] += ' ' + line.strip()
+ else:
+ values['DESCRIPTION'] = line.strip()
else:
indesc = False
- else:
+ if not indesc:
splitline = line.split(':', 1)
- key = line[0]
- value = line[1]
+ if len(splitline) < 2:
+ continue
+ key = splitline[0]
+ value = splitline[1].strip()
if key == 'Build-Depends':
for dep in value.split(','):
dep = dep.split()[0]
mapped = depmap.get(dep, '')
if mapped:
depends.append(mapped)
- elif key == 'Section':
- values['SECTION'] = value
elif key == 'Description':
values['SUMMARY'] = value
indesc = True
+ else:
+ varname = value_map.get(key, None)
+ if varname:
+ values[varname] = value
+ postinst = os.path.join(debpath, 'postinst')
+ postrm = os.path.join(debpath, 'postrm')
+ preinst = os.path.join(debpath, 'preinst')
+ prerm = os.path.join(debpath, 'prerm')
+ sfiles = [postinst, postrm, preinst, prerm]
+ for sfile in sfiles:
+ if os.path.isfile(sfile):
+ logger.info("Converting %s file to recipe function..." %
+ os.path.basename(sfile).upper())
+ content = []
+ with open(sfile) as f:
+ for line in f:
+ if "#!/" in line:
+ continue
+ line = line.rstrip("\n")
+ if line.strip():
+ content.append(line)
+ if content:
+ values[os.path.basename(f.name)] = content
+
+ #if depends:
+ # values['DEPENDS'] = ' '.join(depends)
- if depends:
- values['DEPENDS'] = ' '.join(depends)
+ return values
+
+def convert_rpm_xml(xmlfile):
+ '''Converts the output from rpm -qp --xml to a set of variable values'''
+ import xml.etree.ElementTree as ElementTree
+ rpmtag_map = {'Name': 'PN',
+ 'Version': 'PV',
+ 'Summary': 'SUMMARY',
+ 'Description': 'DESCRIPTION',
+ 'License': 'LICENSE',
+ 'Url': 'HOMEPAGE'}
+ values = {}
+ tree = ElementTree.parse(xmlfile)
+ root = tree.getroot()
+ for child in root:
+ if child.tag == 'rpmTag':
+ name = child.attrib.get('name', None)
+ if name:
+ varname = rpmtag_map.get(name, None)
+ if varname:
+ values[varname] = child[0].text
return values
-def register_command(subparsers):
+def check_npm(tinfoil, debugonly=False):
+ try:
+ rd = tinfoil.parse_recipe('nodejs-native')
+ except bb.providers.NoProvider:
+ # We still conditionally show the message and exit with the special
+ # return code, otherwise we can't show the proper message for eSDK
+ # users
+ log_error_cond('nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs', debugonly)
+ sys.exit(14)
+ bindir = rd.getVar('STAGING_BINDIR_NATIVE')
+ npmpath = os.path.join(bindir, 'npm')
+ if not os.path.exists(npmpath):
+ log_error_cond('npm required to process specified source, but npm is not available - you need to run bitbake -c addto_recipe_sysroot nodejs-native first', debugonly)
+ sys.exit(14)
+ return bindir
+
+def register_commands(subparsers):
parser_create = subparsers.add_parser('create',
help='Create a new recipe',
description='Creates a new recipe from a source tree')
parser_create.add_argument('source', help='Path or URL to source')
- parser_create.add_argument('-o', '--outfile', help='Specify filename for recipe to create', required=True)
+ parser_create.add_argument('-o', '--outfile', help='Specify filename for recipe to create')
parser_create.add_argument('-m', '--machine', help='Make recipe machine-specific as opposed to architecture-specific', action='store_true')
parser_create.add_argument('-x', '--extract-to', metavar='EXTRACTPATH', help='Assuming source is a URL, fetch it and extract it to the directory specified as %(metavar)s')
- parser_create.set_defaults(func=create_recipe)
+ parser_create.add_argument('-N', '--name', help='Name to use within recipe (PN)')
+ parser_create.add_argument('-V', '--version', help='Version to use within recipe (PV)')
+ parser_create.add_argument('-b', '--binary', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure)', action='store_true')
+ parser_create.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true')
+ parser_create.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
+ parser_create.add_argument('-a', '--autorev', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
+ parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
+ parser_create.add_argument('--fetch-dev', action="store_true", help='For npm, also fetch devDependencies')
+ parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS)
+ # FIXME I really hate having to set parserecipes for this, but given we may need
+ # to call into npm (and we don't know in advance if we will or not) and in order
+ # to do so we need to know npm's recipe sysroot path, there's not much alternative
+ parser_create.set_defaults(func=create_recipe, parserecipes=True)
diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py
index 6c9e0efa2a..e914e53aab 100644
--- a/scripts/lib/recipetool/create_buildsys.py
+++ b/scripts/lib/recipetool/create_buildsys.py
@@ -1,6 +1,6 @@
# Recipe creation tool - create command build system handlers
#
-# Copyright (C) 2014 Intel Corporation
+# Copyright (C) 2014-2016 Intel Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -17,23 +17,35 @@
import re
import logging
-from recipetool.create import RecipeHandler, read_pkgconfig_provides
+import glob
+from recipetool.create import RecipeHandler, validate_pv
logger = logging.getLogger('recipetool')
tinfoil = None
+plugins = None
+
+def plugin_init(pluginlist):
+ # Take a reference to the list so we can use it later
+ global plugins
+ plugins = pluginlist
def tinfoil_init(instance):
global tinfoil
tinfoil = instance
+
class CmakeRecipeHandler(RecipeHandler):
- def process(self, srctree, classes, lines_before, lines_after, handled):
+ def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
if 'buildsystem' in handled:
return False
if RecipeHandler.checkfiles(srctree, ['CMakeLists.txt']):
classes.append('cmake')
+ values = CmakeRecipeHandler.extract_cmake_deps(lines_before, srctree, extravalues)
+ classes.extend(values.pop('inherit', '').split())
+ for var, value in values.items():
+ lines_before.append('%s = "%s"' % (var, value))
lines_after.append('# Specify any options you want to pass to cmake using EXTRA_OECMAKE:')
lines_after.append('EXTRA_OECMAKE = ""')
lines_after.append('')
@@ -41,8 +53,266 @@ class CmakeRecipeHandler(RecipeHandler):
return True
return False
+ @staticmethod
+ def extract_cmake_deps(outlines, srctree, extravalues, cmakelistsfile=None):
+ # Find all plugins that want to register handlers
+ logger.debug('Loading cmake handlers')
+ handlers = []
+ for plugin in plugins:
+ if hasattr(plugin, 'register_cmake_handlers'):
+ plugin.register_cmake_handlers(handlers)
+
+ values = {}
+ inherits = []
+
+ if cmakelistsfile:
+ srcfiles = [cmakelistsfile]
+ else:
+ srcfiles = RecipeHandler.checkfiles(srctree, ['CMakeLists.txt'])
+
+ # Note that some of these are non-standard, but probably better to
+ # be able to map them anyway if we see them
+ cmake_pkgmap = {'alsa': 'alsa-lib',
+ 'aspell': 'aspell',
+ 'atk': 'atk',
+ 'bison': 'bison-native',
+ 'boost': 'boost',
+ 'bzip2': 'bzip2',
+ 'cairo': 'cairo',
+ 'cups': 'cups',
+ 'curl': 'curl',
+ 'curses': 'ncurses',
+ 'cvs': 'cvs',
+ 'drm': 'libdrm',
+ 'dbus': 'dbus',
+ 'dbusglib': 'dbus-glib',
+ 'egl': 'virtual/egl',
+ 'expat': 'expat',
+ 'flex': 'flex-native',
+ 'fontconfig': 'fontconfig',
+ 'freetype': 'freetype',
+ 'gettext': '',
+ 'git': '',
+ 'gio': 'glib-2.0',
+ 'giounix': 'glib-2.0',
+ 'glew': 'glew',
+ 'glib': 'glib-2.0',
+ 'glib2': 'glib-2.0',
+ 'glu': 'libglu',
+ 'glut': 'freeglut',
+ 'gobject': 'glib-2.0',
+ 'gperf': 'gperf-native',
+ 'gnutls': 'gnutls',
+ 'gtk2': 'gtk+',
+ 'gtk3': 'gtk+3',
+ 'gtk': 'gtk+3',
+ 'harfbuzz': 'harfbuzz',
+ 'icu': 'icu',
+ 'intl': 'virtual/libintl',
+ 'jpeg': 'jpeg',
+ 'libarchive': 'libarchive',
+ 'libiconv': 'virtual/libiconv',
+ 'liblzma': 'xz',
+ 'libxml2': 'libxml2',
+ 'libxslt': 'libxslt',
+ 'opengl': 'virtual/libgl',
+ 'openmp': '',
+ 'openssl': 'openssl',
+ 'pango': 'pango',
+ 'perl': '',
+ 'perllibs': '',
+ 'pkgconfig': '',
+ 'png': 'libpng',
+ 'pthread': '',
+ 'pythoninterp': '',
+ 'pythonlibs': '',
+ 'ruby': 'ruby-native',
+ 'sdl': 'libsdl',
+ 'sdl2': 'libsdl2',
+ 'subversion': 'subversion-native',
+ 'swig': 'swig-native',
+ 'tcl': 'tcl-native',
+ 'threads': '',
+ 'tiff': 'tiff',
+ 'wget': 'wget',
+ 'x11': 'libx11',
+ 'xcb': 'libxcb',
+ 'xext': 'libxext',
+ 'xfixes': 'libxfixes',
+ 'zlib': 'zlib',
+ }
+
+ pcdeps = []
+ libdeps = []
+ deps = []
+ unmappedpkgs = []
+
+ proj_re = re.compile('project\s*\(([^)]*)\)', re.IGNORECASE)
+ pkgcm_re = re.compile('pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE)
+ pkgsm_re = re.compile('pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE)
+ findpackage_re = re.compile('find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE)
+ findlibrary_re = re.compile('find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*')
+ checklib_re = re.compile('check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE)
+ include_re = re.compile('include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE)
+ subdir_re = re.compile('add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE)
+ dep_re = re.compile('([^ ><=]+)( *[<>=]+ *[^ ><=]+)?')
+
+ def find_cmake_package(pkg):
+ RecipeHandler.load_devel_filemap(tinfoil.config_data)
+ for fn, pn in RecipeHandler.recipecmakefilemap.items():
+ splitname = fn.split('/')
+ if len(splitname) > 1:
+ if splitname[0].lower().startswith(pkg.lower()):
+ if splitname[1] == '%s-config.cmake' % pkg.lower() or splitname[1] == '%sConfig.cmake' % pkg or splitname[1] == 'Find%s.cmake' % pkg:
+ return pn
+ return None
+
+ def interpret_value(value):
+ return value.strip('"')
+
+ def parse_cmake_file(fn, paths=None):
+ searchpaths = (paths or []) + [os.path.dirname(fn)]
+ logger.debug('Parsing file %s' % fn)
+ with open(fn, 'r', errors='surrogateescape') as f:
+ for line in f:
+ line = line.strip()
+ for handler in handlers:
+ if handler.process_line(srctree, fn, line, libdeps, pcdeps, deps, outlines, inherits, values):
+ continue
+ res = include_re.match(line)
+ if res:
+ includefn = bb.utils.which(':'.join(searchpaths), res.group(1))
+ if includefn:
+ parse_cmake_file(includefn, searchpaths)
+ else:
+ logger.debug('Unable to recurse into include file %s' % res.group(1))
+ continue
+ res = subdir_re.match(line)
+ if res:
+ subdirfn = os.path.join(os.path.dirname(fn), res.group(1), 'CMakeLists.txt')
+ if os.path.exists(subdirfn):
+ parse_cmake_file(subdirfn, searchpaths)
+ else:
+ logger.debug('Unable to recurse into subdirectory file %s' % subdirfn)
+ continue
+ res = proj_re.match(line)
+ if res:
+ extravalues['PN'] = interpret_value(res.group(1).split()[0])
+ continue
+ res = pkgcm_re.match(line)
+ if res:
+ res = dep_re.findall(res.group(2))
+ if res:
+ pcdeps.extend([interpret_value(x[0]) for x in res])
+ inherits.append('pkgconfig')
+ continue
+ res = pkgsm_re.match(line)
+ if res:
+ res = dep_re.findall(res.group(2))
+ if res:
+ # Note: appending a tuple here!
+ item = tuple((interpret_value(x[0]) for x in res))
+ if len(item) == 1:
+ item = item[0]
+ pcdeps.append(item)
+ inherits.append('pkgconfig')
+ continue
+ res = findpackage_re.match(line)
+ if res:
+ origpkg = res.group(1)
+ pkg = interpret_value(origpkg)
+ found = False
+ for handler in handlers:
+ if handler.process_findpackage(srctree, fn, pkg, deps, outlines, inherits, values):
+ logger.debug('Mapped CMake package %s via handler %s' % (pkg, handler.__class__.__name__))
+ found = True
+ break
+ if found:
+ continue
+ elif pkg == 'Gettext':
+ inherits.append('gettext')
+ elif pkg == 'Perl':
+ inherits.append('perlnative')
+ elif pkg == 'PkgConfig':
+ inherits.append('pkgconfig')
+ elif pkg == 'PythonInterp':
+ inherits.append('pythonnative')
+ elif pkg == 'PythonLibs':
+ inherits.append('python-dir')
+ else:
+ # Try to map via looking at installed CMake packages in pkgdata
+ dep = find_cmake_package(pkg)
+ if dep:
+ logger.debug('Mapped CMake package %s to recipe %s via pkgdata' % (pkg, dep))
+ deps.append(dep)
+ else:
+ dep = cmake_pkgmap.get(pkg.lower(), None)
+ if dep:
+ logger.debug('Mapped CMake package %s to recipe %s via internal list' % (pkg, dep))
+ deps.append(dep)
+ elif dep is None:
+ unmappedpkgs.append(origpkg)
+ continue
+ res = checklib_re.match(line)
+ if res:
+ lib = interpret_value(res.group(1))
+ if not lib.startswith('$'):
+ libdeps.append(lib)
+ res = findlibrary_re.match(line)
+ if res:
+ libs = res.group(2).split()
+ for lib in libs:
+ if lib in ['HINTS', 'PATHS', 'PATH_SUFFIXES', 'DOC', 'NAMES_PER_DIR'] or lib.startswith(('NO_', 'CMAKE_', 'ONLY_CMAKE_')):
+ break
+ lib = interpret_value(lib)
+ if not lib.startswith('$'):
+ libdeps.append(lib)
+ if line.lower().startswith('useswig'):
+ deps.append('swig-native')
+ continue
+
+ parse_cmake_file(srcfiles[0])
+
+ if unmappedpkgs:
+ outlines.append('# NOTE: unable to map the following CMake package dependencies: %s' % ' '.join(list(set(unmappedpkgs))))
+
+ RecipeHandler.handle_depends(libdeps, pcdeps, deps, outlines, values, tinfoil.config_data)
+
+ for handler in handlers:
+ handler.post_process(srctree, libdeps, pcdeps, deps, outlines, inherits, values)
+
+ if inherits:
+ values['inherit'] = ' '.join(list(set(inherits)))
+
+ return values
+
+
+class CmakeExtensionHandler(object):
+ '''Base class for CMake extension handlers'''
+ def process_line(self, srctree, fn, line, libdeps, pcdeps, deps, outlines, inherits, values):
+ '''
+ Handle a line parsed out of an CMake file.
+ Return True if you've completely handled the passed in line, otherwise return False.
+ '''
+ return False
+
+ def process_findpackage(self, srctree, fn, pkg, deps, outlines, inherits, values):
+ '''
+ Handle a find_package package parsed out of a CMake file.
+ Return True if you've completely handled the passed in package, otherwise return False.
+ '''
+ return False
+
+ def post_process(self, srctree, fn, pkg, deps, outlines, inherits, values):
+ '''
+ Apply any desired post-processing on the output
+ '''
+ return
+
+
+
class SconsRecipeHandler(RecipeHandler):
- def process(self, srctree, classes, lines_before, lines_after, handled):
+ def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
if 'buildsystem' in handled:
return False
@@ -55,8 +325,9 @@ class SconsRecipeHandler(RecipeHandler):
return True
return False
+
class QmakeRecipeHandler(RecipeHandler):
- def process(self, srctree, classes, lines_before, lines_after, handled):
+ def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
if 'buildsystem' in handled:
return False
@@ -66,29 +337,47 @@ class QmakeRecipeHandler(RecipeHandler):
return True
return False
+
class AutotoolsRecipeHandler(RecipeHandler):
- def process(self, srctree, classes, lines_before, lines_after, handled):
+ def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
if 'buildsystem' in handled:
return False
autoconf = False
if RecipeHandler.checkfiles(srctree, ['configure.ac', 'configure.in']):
autoconf = True
- values = AutotoolsRecipeHandler.extract_autotools_deps(lines_before, srctree)
+ values = AutotoolsRecipeHandler.extract_autotools_deps(lines_before, srctree, extravalues)
classes.extend(values.pop('inherit', '').split())
- for var, value in values.iteritems():
+ for var, value in values.items():
lines_before.append('%s = "%s"' % (var, value))
else:
conffile = RecipeHandler.checkfiles(srctree, ['configure'])
if conffile:
# Check if this is just a pre-generated autoconf configure script
- with open(conffile[0], 'r') as f:
+ with open(conffile[0], 'r', errors='surrogateescape') as f:
for i in range(1, 10):
if 'Generated by GNU Autoconf' in f.readline():
autoconf = True
break
+ if autoconf and not ('PV' in extravalues and 'PN' in extravalues):
+ # Last resort
+ conffile = RecipeHandler.checkfiles(srctree, ['configure'])
+ if conffile:
+ with open(conffile[0], 'r', errors='surrogateescape') as f:
+ for line in f:
+ line = line.strip()
+ if line.startswith('VERSION=') or line.startswith('PACKAGE_VERSION='):
+ pv = line.split('=')[1].strip('"\'')
+ if pv and not 'PV' in extravalues and validate_pv(pv):
+ extravalues['PV'] = pv
+ elif line.startswith('PACKAGE_NAME=') or line.startswith('PACKAGE='):
+ pn = line.split('=')[1].strip('"\'')
+ if pn and not 'PN' in extravalues:
+ extravalues['PN'] = pn
+
if autoconf:
+ lines_before.append('')
lines_before.append('# NOTE: if this software is not capable of being built in a separate build directory')
lines_before.append('# from the source, you should replace autotools with autotools-brokensep in the')
lines_before.append('# inherit line')
@@ -102,136 +391,328 @@ class AutotoolsRecipeHandler(RecipeHandler):
return False
@staticmethod
- def extract_autotools_deps(outlines, srctree, acfile=None):
+ def extract_autotools_deps(outlines, srctree, extravalues=None, acfile=None):
import shlex
- import oe.package
+
+ # Find all plugins that want to register handlers
+ logger.debug('Loading autotools handlers')
+ handlers = []
+ for plugin in plugins:
+ if hasattr(plugin, 'register_autotools_handlers'):
+ plugin.register_autotools_handlers(handlers)
values = {}
inherits = []
- # FIXME this mapping is very thin
+ # Hardcoded map, we also use a dynamic one based on what's in the sysroot
progmap = {'flex': 'flex-native',
'bison': 'bison-native',
- 'm4': 'm4-native'}
+ 'm4': 'm4-native',
+ 'tar': 'tar-native',
+ 'ar': 'binutils-native',
+ 'ranlib': 'binutils-native',
+ 'ld': 'binutils-native',
+ 'strip': 'binutils-native',
+ 'libtool': '',
+ 'autoconf': '',
+ 'autoheader': '',
+ 'automake': '',
+ 'uname': '',
+ 'rm': '',
+ 'cp': '',
+ 'mv': '',
+ 'find': '',
+ 'awk': '',
+ 'sed': '',
+ }
progclassmap = {'gconftool-2': 'gconf',
- 'pkg-config': 'pkgconfig'}
-
- ignoredeps = ['gcc-runtime', 'glibc', 'uclibc']
-
- pkg_re = re.compile('PKG_CHECK_MODULES\(\[?[a-zA-Z0-9]*\]?, \[?([^,\]]*)[),].*')
- lib_re = re.compile('AC_CHECK_LIB\(\[?([a-zA-Z0-9]*)\]?, .*')
- progs_re = re.compile('_PROGS?\(\[?[a-zA-Z0-9]*\]?, \[?([^,\]]*)\]?[),].*')
+ 'pkg-config': 'pkgconfig',
+ 'python': 'pythonnative',
+ 'python3': 'python3native',
+ 'perl': 'perlnative',
+ 'makeinfo': 'texinfo',
+ }
+
+ pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
+ pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*')
+ lib_re = re.compile('AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*')
+ libx_re = re.compile('AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*')
+ progs_re = re.compile('_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?')
-
- # Build up lib library->package mapping
- shlib_providers = oe.package.read_shlib_providers(tinfoil.config_data)
- libdir = tinfoil.config_data.getVar('libdir', True)
- base_libdir = tinfoil.config_data.getVar('base_libdir', True)
- libpaths = list(set([base_libdir, libdir]))
- libname_re = re.compile('^lib(.+)\.so.*$')
- pkglibmap = {}
- for lib, item in shlib_providers.iteritems():
- for path, pkg in item.iteritems():
- if path in libpaths:
- res = libname_re.match(lib)
- if res:
- libname = res.group(1)
- if not libname in pkglibmap:
- pkglibmap[libname] = pkg[0]
- else:
- logger.debug('unable to extract library name from %s' % lib)
-
- # Now turn it into a library->recipe mapping
- recipelibmap = {}
- pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True)
- for libname, pkg in pkglibmap.iteritems():
- try:
- with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
- for line in f:
- if line.startswith('PN:'):
- recipelibmap[libname] = line.split(':', 1)[-1].strip()
- break
- except IOError as ioe:
- if ioe.errno == 2:
- logger.warn('unable to find a pkgdata file for package %s' % pkg)
- else:
- raise
+ ac_init_re = re.compile('AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*')
+ am_init_re = re.compile('AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*')
+ define_re = re.compile('\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)')
+ version_re = re.compile('([0-9.]+)')
+
+ defines = {}
+ def subst_defines(value):
+ newvalue = value
+ for define, defval in defines.items():
+ newvalue = newvalue.replace(define, defval)
+ if newvalue != value:
+ return subst_defines(newvalue)
+ return value
+
+ def process_value(value):
+ value = value.replace('[', '').replace(']', '')
+ if value.startswith('m4_esyscmd(') or value.startswith('m4_esyscmd_s('):
+ cmd = subst_defines(value[value.index('(')+1:-1])
+ try:
+ if '|' in cmd:
+ cmd = 'set -o pipefail; ' + cmd
+ stdout, _ = bb.process.run(cmd, cwd=srctree, shell=True)
+ ret = stdout.rstrip()
+ except bb.process.ExecutionError as e:
+ ret = ''
+ elif value.startswith('m4_'):
+ return None
+ ret = subst_defines(value)
+ if ret:
+ ret = ret.strip('"\'')
+ return ret
# Since a configure.ac file is essentially a program, this is only ever going to be
# a hack unfortunately; but it ought to be enough of an approximation
if acfile:
srcfiles = [acfile]
else:
- srcfiles = RecipeHandler.checkfiles(srctree, ['configure.ac', 'configure.in'])
+ srcfiles = RecipeHandler.checkfiles(srctree, ['acinclude.m4', 'configure.ac', 'configure.in'])
+
pcdeps = []
+ libdeps = []
deps = []
unmapped = []
- unmappedlibs = []
- with open(srcfiles[0], 'r') as f:
- for line in f:
- if 'PKG_CHECK_MODULES' in line:
- res = pkg_re.search(line)
+
+ RecipeHandler.load_binmap(tinfoil.config_data)
+
+ def process_macro(keyword, value):
+ for handler in handlers:
+ if handler.process_macro(srctree, keyword, value, process_value, libdeps, pcdeps, deps, outlines, inherits, values):
+ return
+ logger.debug('Found keyword %s with value "%s"' % (keyword, value))
+ if keyword == 'PKG_CHECK_MODULES':
+ res = pkg_re.search(value)
+ if res:
+ res = dep_re.findall(res.group(1))
if res:
- res = dep_re.findall(res.group(1))
- if res:
- pcdeps.extend([x[0] for x in res])
- inherits.append('pkgconfig')
- if line.lstrip().startswith('AM_GNU_GETTEXT'):
- inherits.append('gettext')
- elif 'AC_CHECK_PROG' in line or 'AC_PATH_PROG' in line:
- res = progs_re.search(line)
+ pcdeps.extend([x[0] for x in res])
+ inherits.append('pkgconfig')
+ elif keyword == 'PKG_CHECK_EXISTS':
+ res = pkgce_re.search(value)
+ if res:
+ res = dep_re.findall(res.group(1))
if res:
- for prog in shlex.split(res.group(1)):
- prog = prog.split()[0]
- progclass = progclassmap.get(prog, None)
- if progclass:
- inherits.append(progclass)
- else:
+ pcdeps.extend([x[0] for x in res])
+ inherits.append('pkgconfig')
+ elif keyword in ('AM_GNU_GETTEXT', 'AM_GLIB_GNU_GETTEXT', 'GETTEXT_PACKAGE'):
+ inherits.append('gettext')
+ elif keyword in ('AC_PROG_INTLTOOL', 'IT_PROG_INTLTOOL'):
+ deps.append('intltool-native')
+ elif keyword == 'AM_PATH_GLIB_2_0':
+ deps.append('glib-2.0')
+ elif keyword in ('AC_CHECK_PROG', 'AC_PATH_PROG', 'AX_WITH_PROG'):
+ res = progs_re.search(value)
+ if res:
+ for prog in shlex.split(res.group(1)):
+ prog = prog.split()[0]
+ for handler in handlers:
+ if handler.process_prog(srctree, keyword, value, prog, deps, outlines, inherits, values):
+ return
+ progclass = progclassmap.get(prog, None)
+ if progclass:
+ inherits.append(progclass)
+ else:
+ progdep = RecipeHandler.recipebinmap.get(prog, None)
+ if not progdep:
progdep = progmap.get(prog, None)
- if progdep:
- deps.append(progdep)
- else:
- if not prog.startswith('$'):
- unmapped.append(prog)
- elif 'AC_CHECK_LIB' in line:
- res = lib_re.search(line)
+ if progdep:
+ deps.append(progdep)
+ elif progdep is None:
+ if not prog.startswith('$'):
+ unmapped.append(prog)
+ elif keyword == 'AC_CHECK_LIB':
+ res = lib_re.search(value)
+ if res:
+ lib = res.group(1)
+ if not lib.startswith('$'):
+ libdeps.append(lib)
+ elif keyword == 'AX_CHECK_LIBRARY':
+ res = libx_re.search(value)
+ if res:
+ lib = res.group(2)
+ if not lib.startswith('$'):
+ header = res.group(1)
+ libdeps.append((lib, header))
+ elif keyword == 'AC_PATH_X':
+ deps.append('libx11')
+ elif keyword in ('AX_BOOST', 'BOOST_REQUIRE'):
+ deps.append('boost')
+ elif keyword in ('AC_PROG_LEX', 'AM_PROG_LEX', 'AX_PROG_FLEX'):
+ deps.append('flex-native')
+ elif keyword in ('AC_PROG_YACC', 'AX_PROG_BISON'):
+ deps.append('bison-native')
+ elif keyword == 'AX_CHECK_ZLIB':
+ deps.append('zlib')
+ elif keyword in ('AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO'):
+ deps.append('openssl')
+ elif keyword == 'AX_LIB_CURL':
+ deps.append('curl')
+ elif keyword == 'AX_LIB_BEECRYPT':
+ deps.append('beecrypt')
+ elif keyword == 'AX_LIB_EXPAT':
+ deps.append('expat')
+ elif keyword == 'AX_LIB_GCRYPT':
+ deps.append('libgcrypt')
+ elif keyword == 'AX_LIB_NETTLE':
+ deps.append('nettle')
+ elif keyword == 'AX_LIB_READLINE':
+ deps.append('readline')
+ elif keyword == 'AX_LIB_SQLITE3':
+ deps.append('sqlite3')
+ elif keyword == 'AX_LIB_TAGLIB':
+ deps.append('taglib')
+ elif keyword in ['AX_PKG_SWIG', 'AC_PROG_SWIG']:
+ deps.append('swig-native')
+ elif keyword == 'AX_PROG_XSLTPROC':
+ deps.append('libxslt-native')
+ elif keyword in ['AC_PYTHON_DEVEL', 'AX_PYTHON_DEVEL', 'AM_PATH_PYTHON']:
+ pythonclass = 'pythonnative'
+ res = version_re.search(value)
+ if res:
+ if res.group(1).startswith('3'):
+ pythonclass = 'python3native'
+ # Avoid replacing python3native with pythonnative
+ if not pythonclass in inherits and not 'python3native' in inherits:
+ if 'pythonnative' in inherits:
+ inherits.remove('pythonnative')
+ inherits.append(pythonclass)
+ elif keyword == 'AX_WITH_CURSES':
+ deps.append('ncurses')
+ elif keyword == 'AX_PATH_BDB':
+ deps.append('db')
+ elif keyword == 'AX_PATH_LIB_PCRE':
+ deps.append('libpcre')
+ elif keyword == 'AC_INIT':
+ if extravalues is not None:
+ res = ac_init_re.match(value)
if res:
- lib = res.group(1)
- libdep = recipelibmap.get(lib, None)
- if libdep:
- deps.append(libdep)
- else:
- if libdep is None:
- if not lib.startswith('$'):
- unmappedlibs.append(lib)
- elif 'AC_PATH_X' in line:
- deps.append('libx11')
-
- if unmapped:
- outlines.append('# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(unmapped))
+ extravalues['PN'] = process_value(res.group(1))
+ pv = process_value(res.group(2))
+ if validate_pv(pv):
+ extravalues['PV'] = pv
+ elif keyword == 'AM_INIT_AUTOMAKE':
+ if extravalues is not None:
+ if 'PN' not in extravalues:
+ res = am_init_re.match(value)
+ if res:
+ if res.group(1) != 'AC_PACKAGE_NAME':
+ extravalues['PN'] = process_value(res.group(1))
+ pv = process_value(res.group(2))
+ if validate_pv(pv):
+ extravalues['PV'] = pv
+ elif keyword == 'define(':
+ res = define_re.match(value)
+ if res:
+ key = res.group(2).strip('[]')
+ value = process_value(res.group(3))
+ if value is not None:
+ defines[key] = value
+
+ keywords = ['PKG_CHECK_MODULES',
+ 'PKG_CHECK_EXISTS',
+ 'AM_GNU_GETTEXT',
+ 'AM_GLIB_GNU_GETTEXT',
+ 'GETTEXT_PACKAGE',
+ 'AC_PROG_INTLTOOL',
+ 'IT_PROG_INTLTOOL',
+ 'AM_PATH_GLIB_2_0',
+ 'AC_CHECK_PROG',
+ 'AC_PATH_PROG',
+ 'AX_WITH_PROG',
+ 'AC_CHECK_LIB',
+ 'AX_CHECK_LIBRARY',
+ 'AC_PATH_X',
+ 'AX_BOOST',
+ 'BOOST_REQUIRE',
+ 'AC_PROG_LEX',
+ 'AM_PROG_LEX',
+ 'AX_PROG_FLEX',
+ 'AC_PROG_YACC',
+ 'AX_PROG_BISON',
+ 'AX_CHECK_ZLIB',
+ 'AX_CHECK_OPENSSL',
+ 'AX_LIB_CRYPTO',
+ 'AX_LIB_CURL',
+ 'AX_LIB_BEECRYPT',
+ 'AX_LIB_EXPAT',
+ 'AX_LIB_GCRYPT',
+ 'AX_LIB_NETTLE',
+ 'AX_LIB_READLINE'
+ 'AX_LIB_SQLITE3',
+ 'AX_LIB_TAGLIB',
+ 'AX_PKG_SWIG',
+ 'AC_PROG_SWIG',
+ 'AX_PROG_XSLTPROC',
+ 'AC_PYTHON_DEVEL',
+ 'AX_PYTHON_DEVEL',
+ 'AM_PATH_PYTHON',
+ 'AX_WITH_CURSES',
+ 'AX_PATH_BDB',
+ 'AX_PATH_LIB_PCRE',
+ 'AC_INIT',
+ 'AM_INIT_AUTOMAKE',
+ 'define(',
+ ]
+
+ for handler in handlers:
+ handler.extend_keywords(keywords)
+
+ for srcfile in srcfiles:
+ nesting = 0
+ in_keyword = ''
+ partial = ''
+ with open(srcfile, 'r', errors='surrogateescape') as f:
+ for line in f:
+ if in_keyword:
+ partial += ' ' + line.strip()
+ if partial.endswith('\\'):
+ partial = partial[:-1]
+ nesting = nesting + line.count('(') - line.count(')')
+ if nesting == 0:
+ process_macro(in_keyword, partial)
+ partial = ''
+ in_keyword = ''
+ else:
+ for keyword in keywords:
+ if keyword in line:
+ nesting = line.count('(') - line.count(')')
+ if nesting > 0:
+ partial = line.strip()
+ if partial.endswith('\\'):
+ partial = partial[:-1]
+ in_keyword = keyword
+ else:
+ process_macro(keyword, line.strip())
+ break
- if unmappedlibs:
- outlines.append('# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(unmappedlibs))
- outlines.append('# (this is based on recipes that have previously been built and packaged)')
+ if in_keyword:
+ process_macro(in_keyword, partial)
- recipemap = read_pkgconfig_provides(tinfoil.config_data)
- unmapped = []
- for pcdep in pcdeps:
- recipe = recipemap.get(pcdep, None)
- if recipe:
- deps.append(recipe)
- else:
- if not pcdep.startswith('$'):
- unmapped.append(pcdep)
-
- deps = set(deps).difference(set(ignoredeps))
+ if extravalues:
+ for k,v in list(extravalues.items()):
+ if v:
+ if v.startswith('$') or v.startswith('@') or v.startswith('%'):
+ del extravalues[k]
+ else:
+ extravalues[k] = v.strip('"\'').rstrip('()')
if unmapped:
- outlines.append('# NOTE: unable to map the following pkg-config dependencies: %s' % ' '.join(unmapped))
- outlines.append('# (this is based on recipes that have previously been built and packaged)')
+ outlines.append('# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmapped))))
- if deps:
- values['DEPENDS'] = ' '.join(deps)
+ RecipeHandler.handle_depends(libdeps, pcdeps, deps, outlines, values, tinfoil.config_data)
+
+ for handler in handlers:
+ handler.post_process(srctree, libdeps, pcdeps, deps, outlines, inherits, values)
if inherits:
values['inherit'] = ' '.join(list(set(inherits)))
@@ -239,12 +720,41 @@ class AutotoolsRecipeHandler(RecipeHandler):
return values
+class AutotoolsExtensionHandler(object):
+ '''Base class for Autotools extension handlers'''
+ def process_macro(self, srctree, keyword, value, process_value, libdeps, pcdeps, deps, outlines, inherits, values):
+ '''
+ Handle a macro parsed out of an autotools file. Note that if you want this to be called
+ for any macro other than the ones AutotoolsRecipeHandler already looks for, you'll need
+ to add it to the keywords list in extend_keywords().
+ Return True if you've completely handled the passed in macro, otherwise return False.
+ '''
+ return False
+
+ def extend_keywords(self, keywords):
+ '''Adds keywords to be recognised by the parser (so that you get a call to process_macro)'''
+ return
+
+ def process_prog(self, srctree, keyword, value, prog, deps, outlines, inherits, values):
+ '''
+ Handle an AC_PATH_PROG, AC_CHECK_PROG etc. line
+ Return True if you've completely handled the passed in macro, otherwise return False.
+ '''
+ return False
+
+ def post_process(self, srctree, fn, pkg, deps, outlines, inherits, values):
+ '''
+ Apply any desired post-processing on the output
+ '''
+ return
+
+
class MakefileRecipeHandler(RecipeHandler):
- def process(self, srctree, classes, lines_before, lines_after, handled):
+ def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
if 'buildsystem' in handled:
return False
- makefile = RecipeHandler.checkfiles(srctree, ['Makefile'])
+ makefile = RecipeHandler.checkfiles(srctree, ['Makefile', 'makefile', 'GNUmakefile'])
if makefile:
lines_after.append('# NOTE: this is a Makefile-only piece of software, so we cannot generate much of the')
lines_after.append('# recipe automatically - you will need to examine the Makefile yourself and ensure')
@@ -260,7 +770,7 @@ class MakefileRecipeHandler(RecipeHandler):
if scanfile and os.path.exists(scanfile):
values = AutotoolsRecipeHandler.extract_autotools_deps(lines_before, srctree, acfile=scanfile)
classes.extend(values.pop('inherit', '').split())
- for var, value in values.iteritems():
+ for var, value in values.items():
if var == 'DEPENDS':
lines_before.append('# NOTE: some of these dependencies may be optional, check the Makefile and/or upstream documentation')
lines_before.append('%s = "%s"' % (var, value))
@@ -279,7 +789,7 @@ class MakefileRecipeHandler(RecipeHandler):
installtarget = True
try:
- stdout, stderr = bb.process.run('make -qn install', cwd=srctree, shell=True)
+ stdout, stderr = bb.process.run('make -n install', cwd=srctree, shell=True)
except bb.process.ExecutionError as e:
if e.exitcode != 1:
installtarget = False
@@ -287,7 +797,7 @@ class MakefileRecipeHandler(RecipeHandler):
if installtarget:
func.append('# This is a guess; additional arguments may be required')
makeargs = ''
- with open(makefile[0], 'r') as f:
+ with open(makefile[0], 'r', errors='surrogateescape') as f:
for i in range(1, 100):
if 'DESTDIR' in f.readline():
makeargs += " 'DESTDIR=${D}'"
@@ -307,13 +817,73 @@ class MakefileRecipeHandler(RecipeHandler):
self.genfunction(lines_after, 'do_install', ['# Specify install commands here'])
-def plugin_init(pluginlist):
- pass
+class VersionFileRecipeHandler(RecipeHandler):
+ def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+ if 'PV' not in extravalues:
+ # Look for a VERSION or version file containing a single line consisting
+ # only of a version number
+ filelist = RecipeHandler.checkfiles(srctree, ['VERSION', 'version'])
+ version = None
+ for fileitem in filelist:
+ linecount = 0
+ with open(fileitem, 'r', errors='surrogateescape') as f:
+ for line in f:
+ line = line.rstrip().strip('"\'')
+ linecount += 1
+ if line:
+ if linecount > 1:
+ version = None
+ break
+ else:
+ if validate_pv(line):
+ version = line
+ if version:
+ extravalues['PV'] = version
+ break
+
+
+class SpecFileRecipeHandler(RecipeHandler):
+ def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+ if 'PV' in extravalues and 'PN' in extravalues:
+ return
+ filelist = RecipeHandler.checkfiles(srctree, ['*.spec'], recursive=True)
+ valuemap = {'Name': 'PN',
+ 'Version': 'PV',
+ 'Summary': 'SUMMARY',
+ 'Url': 'HOMEPAGE',
+ 'License': 'LICENSE'}
+ foundvalues = {}
+ for fileitem in filelist:
+ linecount = 0
+ with open(fileitem, 'r', errors='surrogateescape') as f:
+ for line in f:
+ for value, varname in valuemap.items():
+ if line.startswith(value + ':') and not varname in foundvalues:
+ foundvalues[varname] = line.split(':', 1)[1].strip()
+ break
+ if len(foundvalues) == len(valuemap):
+ break
+ if 'PV' in foundvalues:
+ if not validate_pv(foundvalues['PV']):
+ del foundvalues['PV']
+ license = foundvalues.pop('LICENSE', None)
+ if license:
+ liccomment = '# NOTE: spec file indicates the license may be "%s"' % license
+ for i, line in enumerate(lines_before):
+ if line.startswith('LICENSE ='):
+ lines_before.insert(i, liccomment)
+ break
+ else:
+ lines_before.append(liccomment)
+ extravalues.update(foundvalues)
def register_recipe_handlers(handlers):
- # These are in a specific order so that the right one is detected first
- handlers.append(CmakeRecipeHandler())
- handlers.append(AutotoolsRecipeHandler())
- handlers.append(SconsRecipeHandler())
- handlers.append(QmakeRecipeHandler())
- handlers.append(MakefileRecipeHandler())
+ # Set priorities with some gaps so that other plugins can insert
+ # their own handlers (so avoid changing these numbers)
+ handlers.append((CmakeRecipeHandler(), 50))
+ handlers.append((AutotoolsRecipeHandler(), 40))
+ handlers.append((SconsRecipeHandler(), 30))
+ handlers.append((QmakeRecipeHandler(), 20))
+ handlers.append((MakefileRecipeHandler(), 10))
+ handlers.append((VersionFileRecipeHandler(), -1))
+ handlers.append((SpecFileRecipeHandler(), -1))
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py
index f4f4212559..ec5449bee9 100644
--- a/scripts/lib/recipetool/create_buildsys_python.py
+++ b/scripts/lib/recipetool/create_buildsys_python.py
@@ -61,8 +61,6 @@ class PythonRecipeHandler(RecipeHandler):
}
# PN/PV are already set by recipetool core & desc can be extremely long
excluded_fields = [
- 'Name',
- 'Version',
'Description',
]
setup_parse_map = {
@@ -88,8 +86,11 @@ class PythonRecipeHandler(RecipeHandler):
]
setuparg_multi_line_values = ['Description']
replacements = [
+ ('License', r' +$', ''),
+ ('License', r'^ +', ''),
('License', r' ', '-'),
- ('License', r'-License$', ''),
+ ('License', r'^GNU-', ''),
+ ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''),
('License', r'^UNKNOWN$', ''),
# Remove currently unhandled version numbers from these variables
@@ -159,7 +160,7 @@ class PythonRecipeHandler(RecipeHandler):
def __init__(self):
pass
- def process(self, srctree, classes, lines_before, lines_after, handled):
+ def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
if 'buildsystem' in handled:
return False
@@ -218,6 +219,9 @@ class PythonRecipeHandler(RecipeHandler):
else:
info = self.get_setup_args_info(setupscript)
+ # Grab the license value before applying replacements
+ license_str = info.get('License', '').strip()
+
self.apply_info_replacements(info)
if uses_setuptools:
@@ -225,60 +229,53 @@ class PythonRecipeHandler(RecipeHandler):
else:
classes.append('distutils')
+ if license_str:
+ for i, line in enumerate(lines_before):
+ if line.startswith('LICENSE = '):
+ lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str)
+ break
+
if 'Classifier' in info:
+ existing_licenses = info.get('License', '')
licenses = []
for classifier in info['Classifier']:
if classifier in self.classifier_license_map:
license = self.classifier_license_map[classifier]
+ if license == 'Apache' and 'Apache-2.0' in existing_licenses:
+ license = 'Apache-2.0'
+ elif license == 'GPL':
+ if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
+ license = 'GPL-2.0'
+ elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
+ license = 'GPL-3.0'
+ elif license == 'LGPL':
+ if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
+ license = 'LGPL-2.1'
+ elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
+ license = 'LGPL-2.0'
+ elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
+ license = 'LGPL-3.0'
licenses.append(license)
if licenses:
info['License'] = ' & '.join(licenses)
-
# Map PKG-INFO & setup.py fields to bitbake variables
- bbinfo = {}
- for field, values in info.iteritems():
+ for field, values in info.items():
if field in self.excluded_fields:
continue
if field not in self.bbvar_map:
continue
- if isinstance(values, basestring):
+ if isinstance(values, str):
value = values
else:
value = ' '.join(str(v) for v in values if v)
bbvar = self.bbvar_map[field]
- if bbvar not in bbinfo and value:
- bbinfo[bbvar] = value
-
- comment_lic_line = None
- for pos, line in enumerate(list(lines_before)):
- if line.startswith('#') and 'LICENSE' in line:
- comment_lic_line = pos
- elif line.startswith('LICENSE =') and 'LICENSE' in bbinfo:
- if line in ('LICENSE = "Unknown"', 'LICENSE = "CLOSED"'):
- lines_before[pos] = 'LICENSE = "{}"'.format(bbinfo['LICENSE'])
- if line == 'LICENSE = "CLOSED"' and comment_lic_line:
- lines_before[comment_lic_line:pos] = [
- '# WARNING: the following LICENSE value is a best guess - it is your',
- '# responsibility to verify that the value is complete and correct.'
- ]
- del bbinfo['LICENSE']
-
- src_uri_line = None
- for pos, line in enumerate(lines_before):
- if line.startswith('SRC_URI ='):
- src_uri_line = pos
-
- if bbinfo:
- mdinfo = ['']
- for k in sorted(bbinfo):
- v = bbinfo[k]
- mdinfo.append('{} = "{}"'.format(k, v))
- lines_before[src_uri_line-1:src_uri_line-1] = mdinfo
+ if bbvar not in extravalues and value:
+ extravalues[bbvar] = value
mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
@@ -291,8 +288,8 @@ class PythonRecipeHandler(RecipeHandler):
lines_after.append('# The upstream names may not correspond exactly to bitbake package names.')
lines_after.append('#')
lines_after.append('# Uncomment this line to enable all the optional features.')
- lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req.iterkeys())))
- for feature, feature_reqs in extras_req.iteritems():
+ lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req)))
+ for feature, feature_reqs in extras_req.items():
unmapped_deps.difference_update(feature_reqs)
feature_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(feature_reqs))
@@ -358,7 +355,7 @@ class PythonRecipeHandler(RecipeHandler):
# Naive mapping of setup() arguments to PKG-INFO field names
for d in [info, non_literals]:
- for key, value in d.items():
+ for key, value in list(d.items()):
new_key = _map(key)
if new_key != key:
del d[key]
@@ -433,14 +430,14 @@ class PythonRecipeHandler(RecipeHandler):
return value
value = info[variable]
- if isinstance(value, basestring):
+ if isinstance(value, str):
new_value = replace_value(search, replace, value)
if new_value is None:
del info[variable]
elif new_value != value:
info[variable] = new_value
- elif hasattr(value, 'iteritems'):
- for dkey, dvalue in value.iteritems():
+ elif hasattr(value, 'items'):
+ for dkey, dvalue in list(value.items()):
new_list = []
for pos, a_value in enumerate(dvalue):
new_value = replace_value(search, replace, a_value)
@@ -501,8 +498,10 @@ class PythonRecipeHandler(RecipeHandler):
for dep in scanned_deps:
mapped = provided_packages.get(dep)
if mapped:
+ logger.debug('Mapped %s to %s' % (dep, mapped))
mapped_deps.add(mapped)
else:
+ logger.debug('Could not map %s' % dep)
unmapped_deps.add(dep)
return mapped_deps, unmapped_deps
@@ -533,11 +532,11 @@ class PythonRecipeHandler(RecipeHandler):
def parse_pkgdata_for_python_packages(self):
suffixes = [t[0] for t in imp.get_suffixes()]
- pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True)
+ pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
ldata = tinfoil.config_data.createCopy()
bb.parse.handle('classes/python-dir.bbclass', ldata, True)
- python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR', True)
+ python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
python_dirs = [python_sitedir + os.sep,
@@ -555,7 +554,7 @@ class PythonRecipeHandler(RecipeHandler):
else:
continue
- for fn in files_info.iterkeys():
+ for fn in files_info:
for suffix in suffixes:
if fn.endswith(suffix):
break
@@ -563,6 +562,8 @@ class PythonRecipeHandler(RecipeHandler):
continue
if fn.startswith(dynload_dir + os.sep):
+ if '/.debug/' in fn:
+ continue
base = os.path.basename(fn)
provided = base.split('.', 1)[0]
packages[provided] = os.path.basename(pkgdatafile)
@@ -590,7 +591,7 @@ class PythonRecipeHandler(RecipeHandler):
if 'stderr' not in popenargs:
popenargs['stderr'] = subprocess.STDOUT
try:
- return subprocess.check_output(cmd, **popenargs)
+ return subprocess.check_output(cmd, **popenargs).decode('utf-8')
except OSError as exc:
logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
raise
@@ -605,7 +606,7 @@ def gather_setup_info(fileobj):
visitor.visit(parsed)
non_literals, extensions = {}, []
- for key, value in visitor.keywords.items():
+ for key, value in list(visitor.keywords.items()):
if key == 'ext_modules':
if isinstance(value, list):
for ext in value:
@@ -637,7 +638,7 @@ class SetupScriptVisitor(ast.NodeVisitor):
def visit_setup(self, node):
call = LiteralAstTransform().visit(node)
self.keywords = call.keywords
- for k, v in self.keywords.iteritems():
+ for k, v in self.keywords.items():
if has_non_literals(v):
self.non_literals.append(k)
@@ -703,18 +704,14 @@ class LiteralAstTransform(ast.NodeTransformer):
def has_non_literals(value):
if isinstance(value, ast.AST):
return True
- elif isinstance(value, basestring):
+ elif isinstance(value, str):
return False
- elif hasattr(value, 'itervalues'):
- return any(has_non_literals(v) for v in value.itervalues())
+ elif hasattr(value, 'values'):
+ return any(has_non_literals(v) for v in value.values())
elif hasattr(value, '__iter__'):
return any(has_non_literals(v) for v in value)
-def plugin_init(pluginlist):
- pass
-
-
def register_recipe_handlers(handlers):
# We need to make sure this is ahead of the makefile fallback handler
- handlers.insert(0, PythonRecipeHandler())
+ handlers.append((PythonRecipeHandler(), 70))
diff --git a/scripts/lib/recipetool/create_kernel.py b/scripts/lib/recipetool/create_kernel.py
new file mode 100644
index 0000000000..ca4996c7ac
--- /dev/null
+++ b/scripts/lib/recipetool/create_kernel.py
@@ -0,0 +1,99 @@
+# Recipe creation tool - kernel support plugin
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import re
+import logging
+from recipetool.create import RecipeHandler, read_pkgconfig_provides, validate_pv
+
+logger = logging.getLogger('recipetool')
+
+tinfoil = None
+
+def tinfoil_init(instance):
+ global tinfoil
+ tinfoil = instance
+
+
+class KernelRecipeHandler(RecipeHandler):
+ def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+ import bb.process
+ if 'buildsystem' in handled:
+ return False
+
+ for tell in ['arch', 'firmware', 'Kbuild', 'Kconfig']:
+ if not os.path.exists(os.path.join(srctree, tell)):
+ return False
+
+ handled.append('buildsystem')
+ del lines_after[:]
+ del classes[:]
+ template = os.path.join(tinfoil.config_data.getVar('COREBASE'), 'meta-skeleton', 'recipes-kernel', 'linux', 'linux-yocto-custom.bb')
+ def handle_var(varname, origvalue, op, newlines):
+ if varname in ['SRCREV', 'SRCREV_machine']:
+ while newlines[-1].startswith('#'):
+ del newlines[-1]
+ try:
+ stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree, shell=True)
+ except bb.process.ExecutionError as e:
+ stdout = None
+ if stdout:
+ return stdout.strip(), op, 0, True
+ elif varname == 'LINUX_VERSION':
+ makefile = os.path.join(srctree, 'Makefile')
+ if os.path.exists(makefile):
+ kversion = -1
+ kpatchlevel = -1
+ ksublevel = -1
+ kextraversion = ''
+ with open(makefile, 'r', errors='surrogateescape') as f:
+ for i, line in enumerate(f):
+ if i > 10:
+ break
+ if line.startswith('VERSION ='):
+ kversion = int(line.split('=')[1].strip())
+ elif line.startswith('PATCHLEVEL ='):
+ kpatchlevel = int(line.split('=')[1].strip())
+ elif line.startswith('SUBLEVEL ='):
+ ksublevel = int(line.split('=')[1].strip())
+ elif line.startswith('EXTRAVERSION ='):
+ kextraversion = line.split('=')[1].strip()
+ version = ''
+ if kversion > -1 and kpatchlevel > -1:
+ version = '%d.%d' % (kversion, kpatchlevel)
+ if ksublevel > -1:
+ version += '.%d' % ksublevel
+ version += kextraversion
+ if version:
+ return version, op, 0, True
+ elif varname == 'SRC_URI':
+ while newlines[-1].startswith('#'):
+ del newlines[-1]
+ elif varname == 'COMPATIBLE_MACHINE':
+ while newlines[-1].startswith('#'):
+ del newlines[-1]
+ machine = tinfoil.config_data.getVar('MACHINE')
+ return machine, op, 0, True
+ return origvalue, op, 0, True
+ with open(template, 'r') as f:
+ varlist = ['SRCREV', 'SRCREV_machine', 'SRC_URI', 'LINUX_VERSION', 'COMPATIBLE_MACHINE']
+ (_, newlines) = bb.utils.edit_metadata(f, varlist, handle_var)
+ lines_before[:] = [line.rstrip('\n') for line in newlines]
+
+ return True
+
+def register_recipe_handlers(handlers):
+ handlers.append((KernelRecipeHandler(), 100))
diff --git a/scripts/lib/recipetool/create_kmod.py b/scripts/lib/recipetool/create_kmod.py
new file mode 100644
index 0000000000..7cf188db21
--- /dev/null
+++ b/scripts/lib/recipetool/create_kmod.py
@@ -0,0 +1,152 @@
+# Recipe creation tool - kernel module support plugin
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import re
+import logging
+from recipetool.create import RecipeHandler, read_pkgconfig_provides, validate_pv
+
+logger = logging.getLogger('recipetool')
+
+tinfoil = None
+
+def tinfoil_init(instance):
+ global tinfoil
+ tinfoil = instance
+
+
+class KernelModuleRecipeHandler(RecipeHandler):
+ def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+ import bb.process
+ if 'buildsystem' in handled:
+ return False
+
+ module_inc_re = re.compile(r'^#include\s+<linux/module.h>$')
+ makefiles = []
+ is_module = False
+
+ makefiles = []
+
+ files = RecipeHandler.checkfiles(srctree, ['*.c', '*.h'], recursive=True)
+ if files:
+ for cfile in files:
+ # Look in same dir or parent for Makefile
+ for makefile in [os.path.join(os.path.dirname(cfile), 'Makefile'), os.path.join(os.path.dirname(os.path.dirname(cfile)), 'Makefile')]:
+ if makefile in makefiles:
+ break
+ else:
+ if os.path.exists(makefile):
+ makefiles.append(makefile)
+ break
+ else:
+ continue
+ with open(cfile, 'r', errors='surrogateescape') as f:
+ for line in f:
+ if module_inc_re.match(line.strip()):
+ is_module = True
+ break
+ if is_module:
+ break
+
+ if is_module:
+ classes.append('module')
+ handled.append('buildsystem')
+ # module.bbclass and the classes it inherits do most of the hard
+ # work, but we need to tweak it slightly depending on what the
+ # Makefile does (and there is a range of those)
+ # Check the makefile for the appropriate install target
+ install_lines = []
+ compile_lines = []
+ in_install = False
+ in_compile = False
+ install_target = None
+ with open(makefile, 'r', errors='surrogateescape') as f:
+ for line in f:
+ if line.startswith('install:'):
+ if not install_lines:
+ in_install = True
+ install_target = 'install'
+ elif line.startswith('modules_install:'):
+ install_lines = []
+ in_install = True
+ install_target = 'modules_install'
+ elif line.startswith('modules:'):
+ compile_lines = []
+ in_compile = True
+ elif line.startswith(('all:', 'default:')):
+ if not compile_lines:
+ in_compile = True
+ elif line:
+ if line[0] == '\t':
+ if in_install:
+ install_lines.append(line)
+ elif in_compile:
+ compile_lines.append(line)
+ elif ':' in line:
+ in_install = False
+ in_compile = False
+
+ def check_target(lines, install):
+ kdirpath = ''
+ manual_install = False
+ for line in lines:
+ splitline = line.split()
+ if splitline[0] in ['make', 'gmake', '$(MAKE)']:
+ if '-C' in splitline:
+ idx = splitline.index('-C') + 1
+ if idx < len(splitline):
+ kdirpath = splitline[idx]
+ break
+ elif install and splitline[0] == 'install':
+ if '.ko' in line:
+ manual_install = True
+ return kdirpath, manual_install
+
+ kdirpath = None
+ manual_install = False
+ if install_lines:
+ kdirpath, manual_install = check_target(install_lines, install=True)
+ if compile_lines and not kdirpath:
+ kdirpath, _ = check_target(compile_lines, install=False)
+
+ if manual_install or not install_lines:
+ lines_after.append('EXTRA_OEMAKE_append_task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"')
+ elif install_target and install_target != 'modules_install':
+ lines_after.append('MODULES_INSTALL_TARGET = "install"')
+
+ warnmsg = None
+ kdirvar = None
+ if kdirpath:
+ res = re.match(r'\$\(([^$)]+)\)', kdirpath)
+ if res:
+ kdirvar = res.group(1)
+ if kdirvar != 'KERNEL_SRC':
+ lines_after.append('EXTRA_OEMAKE += "%s=${STAGING_KERNEL_DIR}"' % kdirvar)
+ elif kdirpath.startswith('/lib/'):
+ warnmsg = 'Kernel path in install makefile is hardcoded - you will need to patch the makefile'
+ if not kdirvar and not warnmsg:
+ warnmsg = 'Unable to find means of passing kernel path into install makefile - if kernel path is hardcoded you will need to patch the makefile'
+ if warnmsg:
+ warnmsg += '. Note that the variable KERNEL_SRC will be passed in as the kernel source path.'
+ logger.warn(warnmsg)
+ lines_after.append('# %s' % warnmsg)
+
+ return True
+
+ return False
+
+def register_recipe_handlers(handlers):
+ handlers.append((KernelModuleRecipeHandler(), 15))
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
new file mode 100644
index 0000000000..cb8f338b8b
--- /dev/null
+++ b/scripts/lib/recipetool/create_npm.py
@@ -0,0 +1,344 @@
+# Recipe creation tool - node.js NPM module support plugin
+#
+# Copyright (C) 2016 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import logging
+import subprocess
+import tempfile
+import shutil
+import json
+from recipetool.create import RecipeHandler, split_pkg_licenses, handle_license_vars, check_npm
+
+logger = logging.getLogger('recipetool')
+
+
+tinfoil = None
+
+def tinfoil_init(instance):
+ global tinfoil
+ tinfoil = instance
+
+
+class NpmRecipeHandler(RecipeHandler):
+ lockdownpath = None
+
+ def _handle_license(self, data):
+ '''
+ Handle the license value from an npm package.json file
+ '''
+ license = None
+ if 'license' in data:
+ license = data['license']
+ if isinstance(license, dict):
+ license = license.get('type', None)
+ if license:
+ if 'OR' in license:
+ license = license.replace('OR', '|')
+ license = license.replace('AND', '&')
+ license = license.replace(' ', '_')
+ if not license[0] == '(':
+ license = '(' + license + ')'
+ print('LICENSE: {}'.format(license))
+ else:
+ license = license.replace('AND', '&')
+ if license[0] == '(':
+ license = license[1:]
+ if license[-1] == ')':
+ license = license[:-1]
+ license = license.replace('MIT/X11', 'MIT')
+ license = license.replace('Public Domain', 'PD')
+ license = license.replace('SEE LICENSE IN EULA',
+ 'SEE-LICENSE-IN-EULA')
+ return license
+
+ def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d):
+ try:
+ runenv = dict(os.environ, PATH=d.getVar('PATH'))
+ bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
+ except bb.process.ExecutionError as e:
+ logger.warn('npm shrinkwrap failed:\n%s' % e.stdout)
+ return
+
+ tmpfile = os.path.join(localfilesdir, 'npm-shrinkwrap.json')
+ shutil.move(os.path.join(srctree, 'npm-shrinkwrap.json'), tmpfile)
+ extravalues.setdefault('extrafiles', {})
+ extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
+ lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
+
+ def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d):
+ runenv = dict(os.environ, PATH=d.getVar('PATH'))
+ if not NpmRecipeHandler.lockdownpath:
+ NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
+ bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
+ cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
+ relockbin = os.path.join(NpmRecipeHandler.lockdownpath, 'node_modules', 'lockdown', 'relock.js')
+ if not os.path.exists(relockbin):
+ logger.warn('Could not find relock.js within lockdown directory; skipping lockdown')
+ return
+ try:
+ bb.process.run('node %s' % relockbin, cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
+ except bb.process.ExecutionError as e:
+ logger.warn('lockdown-relock failed:\n%s' % e.stdout)
+ return
+
+ tmpfile = os.path.join(localfilesdir, 'lockdown.json')
+ shutil.move(os.path.join(srctree, 'lockdown.json'), tmpfile)
+ extravalues.setdefault('extrafiles', {})
+ extravalues['extrafiles']['lockdown.json'] = tmpfile
+ lines_before.append('NPM_LOCKDOWN := "${THISDIR}/${PN}/lockdown.json"')
+
+ def _handle_dependencies(self, d, deps, optdeps, devdeps, lines_before, srctree):
+ import scriptutils
+ # If this isn't a single module we need to get the dependencies
+ # and add them to SRC_URI
+ def varfunc(varname, origvalue, op, newlines):
+ if varname == 'SRC_URI':
+ if not origvalue.startswith('npm://'):
+ src_uri = origvalue.split()
+ changed = False
+ deplist = {}
+ for dep, depver in optdeps.items():
+ depdata = self.get_npm_data(dep, depver, d)
+ if self.check_npm_optional_dependency(depdata):
+ deplist[dep] = depdata
+ for dep, depver in devdeps.items():
+ depdata = self.get_npm_data(dep, depver, d)
+ if self.check_npm_optional_dependency(depdata):
+ deplist[dep] = depdata
+ for dep, depver in deps.items():
+ depdata = self.get_npm_data(dep, depver, d)
+ deplist[dep] = depdata
+
+ for dep, depdata in deplist.items():
+ version = depdata.get('version', None)
+ if version:
+ url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
+ scriptutils.fetch_uri(d, url, srctree)
+ src_uri.append(url)
+ changed = True
+ if changed:
+ return src_uri, None, -1, True
+ return origvalue, None, 0, True
+ updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
+ if updated:
+ del lines_before[:]
+ for line in newlines:
+ # Hack to avoid newlines that edit_metadata inserts
+ if line.endswith('\n'):
+ line = line[:-1]
+ lines_before.append(line)
+ return updated
+
+ def _replace_license_vars(self, srctree, lines_before, handled, extravalues, d):
+ for item in handled:
+ if isinstance(item, tuple):
+ if item[0] == 'license':
+ del item
+ break
+
+ calledvars = []
+ def varfunc(varname, origvalue, op, newlines):
+ if varname in ['LICENSE', 'LIC_FILES_CHKSUM']:
+ for i, e in enumerate(reversed(newlines)):
+ if not e.startswith('#'):
+ stop = i
+ while stop > 0:
+ newlines.pop()
+ stop -= 1
+ break
+ calledvars.append(varname)
+ if len(calledvars) > 1:
+ # The second time around, put the new license text in
+ insertpos = len(newlines)
+ handle_license_vars(srctree, newlines, handled, extravalues, d)
+ return None, None, 0, True
+ return origvalue, None, 0, True
+ updated, newlines = bb.utils.edit_metadata(lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc)
+ if updated:
+ del lines_before[:]
+ lines_before.extend(newlines)
+ else:
+ raise Exception('Did not find license variables')
+
+ def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+ import bb.utils
+ import oe
+ from collections import OrderedDict
+
+ if 'buildsystem' in handled:
+ return False
+
+ def read_package_json(fn):
+ with open(fn, 'r', errors='surrogateescape') as f:
+ return json.loads(f.read())
+
+ files = RecipeHandler.checkfiles(srctree, ['package.json'])
+ if files:
+ d = bb.data.createCopy(tinfoil.config_data)
+ npm_bindir = check_npm(tinfoil, self._devtool)
+ d.prependVar('PATH', '%s:' % npm_bindir)
+
+ data = read_package_json(files[0])
+ if 'name' in data and 'version' in data:
+ extravalues['PN'] = data['name']
+ extravalues['PV'] = data['version']
+ classes.append('npm')
+ handled.append('buildsystem')
+ if 'description' in data:
+ extravalues['SUMMARY'] = data['description']
+ if 'homepage' in data:
+ extravalues['HOMEPAGE'] = data['homepage']
+
+ fetchdev = extravalues['fetchdev'] or None
+ deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
+ updated = self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)
+ if updated:
+ # We need to redo the license stuff
+ self._replace_license_vars(srctree, lines_before, handled, extravalues, d)
+
+ # Shrinkwrap
+ localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
+ self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d)
+
+ # Lockdown
+ self._lockdown(srctree, localfilesdir, extravalues, lines_before, d)
+
+ # Split each npm module out to is own package
+ npmpackages = oe.package.npm_split_package_dirs(srctree)
+ for item in handled:
+ if isinstance(item, tuple):
+ if item[0] == 'license':
+ licvalues = item[1]
+ break
+ if licvalues:
+ # Augment the license list with information we have in the packages
+ licenses = {}
+ license = self._handle_license(data)
+ if license:
+ licenses['${PN}'] = license
+ for pkgname, pkgitem in npmpackages.items():
+ _, pdata = pkgitem
+ license = self._handle_license(pdata)
+ if license:
+ licenses[pkgname] = license
+ # Now write out the package-specific license values
+ # We need to strip out the json data dicts for this since split_pkg_licenses
+ # isn't expecting it
+ packages = OrderedDict((x,y[0]) for x,y in npmpackages.items())
+ packages['${PN}'] = ''
+ pkglicenses = split_pkg_licenses(licvalues, packages, lines_after, licenses)
+ all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense]))
+ if '&' in all_licenses:
+ all_licenses.remove('&')
+ # Go back and update the LICENSE value since we have a bit more
+ # information than when that was written out (and we know all apply
+ # vs. there being a choice, so we can join them with &)
+ for i, line in enumerate(lines_before):
+ if line.startswith('LICENSE = '):
+ lines_before[i] = 'LICENSE = "%s"' % ' & '.join(all_licenses)
+ break
+
+ # Need to move S setting after inherit npm
+ for i, line in enumerate(lines_before):
+ if line.startswith('S ='):
+ lines_before.pop(i)
+ lines_after.insert(0, '# Must be set after inherit npm since that itself sets S')
+ lines_after.insert(1, line)
+ break
+
+ return True
+
+ return False
+
+ # FIXME this is duplicated from lib/bb/fetch2/npm.py
+ def _parse_view(self, output):
+ '''
+ Parse the output of npm view --json; the last JSON result
+ is assumed to be the one that we're interested in.
+ '''
+ pdata = None
+ outdeps = {}
+ datalines = []
+ bracelevel = 0
+ for line in output.splitlines():
+ if bracelevel:
+ datalines.append(line)
+ elif '{' in line:
+ datalines = []
+ datalines.append(line)
+ bracelevel = bracelevel + line.count('{') - line.count('}')
+ if datalines:
+ pdata = json.loads('\n'.join(datalines))
+ return pdata
+
+ # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
+ # (split out from _getdependencies())
+ def get_npm_data(self, pkg, version, d):
+ import bb.fetch2
+ pkgfullname = pkg
+ if version != '*' and not '/' in version:
+ pkgfullname += "@'%s'" % version
+ logger.debug(2, "Calling getdeps on %s" % pkg)
+ runenv = dict(os.environ, PATH=d.getVar('PATH'))
+ fetchcmd = "npm view %s --json" % pkgfullname
+ output, _ = bb.process.run(fetchcmd, stderr=subprocess.STDOUT, env=runenv, shell=True)
+ data = self._parse_view(output)
+ return data
+
+ # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
+ # (split out from _getdependencies())
+ def get_npm_package_dependencies(self, pdata, fetchdev):
+ dependencies = pdata.get('dependencies', {})
+ optionalDependencies = pdata.get('optionalDependencies', {})
+ dependencies.update(optionalDependencies)
+ if fetchdev:
+ devDependencies = pdata.get('devDependencies', {})
+ dependencies.update(devDependencies)
+ else:
+ devDependencies = {}
+ depsfound = {}
+ optdepsfound = {}
+ devdepsfound = {}
+ for dep in dependencies:
+ if dep in optionalDependencies:
+ optdepsfound[dep] = dependencies[dep]
+ elif dep in devDependencies:
+ devdepsfound[dep] = dependencies[dep]
+ else:
+ depsfound[dep] = dependencies[dep]
+ return depsfound, optdepsfound, devdepsfound
+
+ # FIXME this is effectively duplicated from lib/bb/fetch2/npm.py
+ # (split out from _getdependencies())
+ def check_npm_optional_dependency(self, pdata):
+ pkg_os = pdata.get('os', None)
+ if pkg_os:
+ if not isinstance(pkg_os, list):
+ pkg_os = [pkg_os]
+ blacklist = False
+ for item in pkg_os:
+ if item.startswith('!'):
+ blacklist = True
+ break
+ if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os:
+ logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg)
+ return False
+ return True
+
+
+def register_recipe_handlers(handlers):
+ handlers.append((NpmRecipeHandler(), 60))
diff --git a/scripts/lib/recipetool/newappend.py b/scripts/lib/recipetool/newappend.py
new file mode 100644
index 0000000000..0b63759d8c
--- /dev/null
+++ b/scripts/lib/recipetool/newappend.py
@@ -0,0 +1,89 @@
+# Recipe creation tool - newappend plugin
+#
+# This sub-command creates a bbappend for the specified target and prints the
+# path to the bbappend.
+#
+# Example: recipetool newappend meta-mylayer busybox
+#
+# Copyright (C) 2015 Christopher Larson <kergoth@gmail.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import argparse
+import errno
+import logging
+import os
+import re
+import subprocess
+import sys
+import scriptutils
+
+
+logger = logging.getLogger('recipetool')
+tinfoil = None
+
+
+def tinfoil_init(instance):
+ global tinfoil
+ tinfoil = instance
+
+
+def layer(layerpath):
+ if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')):
+ raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath))
+ return layerpath
+
+
+def newappend(args):
+ import oe.recipeutils
+
+ recipe_path = tinfoil.get_recipe_file(args.target)
+
+ rd = tinfoil.config_data.createCopy()
+ rd.setVar('FILE', recipe_path)
+ append_path, path_ok = oe.recipeutils.get_bbappend_path(rd, args.destlayer, args.wildcard_version)
+ if not append_path:
+ logger.error('Unable to determine layer directory containing %s', recipe_path)
+ return 1
+
+ if not path_ok:
+ logger.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.', os.path.join(args.destlayer, 'conf', 'layer.conf'), os.path.dirname(append_path))
+
+ layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
+ if not os.path.abspath(args.destlayer) in layerdirs:
+ logger.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active')
+
+ if not os.path.exists(append_path):
+ bb.utils.mkdirhier(os.path.dirname(append_path))
+
+ try:
+ open(append_path, 'a').close()
+ except (OSError, IOError) as exc:
+ logger.critical(str(exc))
+ return 1
+
+ if args.edit:
+ return scriptutils.run_editor([append_path, recipe_path])
+ else:
+ print(append_path)
+
+
+def register_commands(subparsers):
+ parser = subparsers.add_parser('newappend',
+ help='Create a bbappend for the specified target in the specified layer')
+ parser.add_argument('-e', '--edit', help='Edit the new append. This obeys $VISUAL if set, otherwise $EDITOR, otherwise vi.', action='store_true')
+ parser.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true')
+ parser.add_argument('destlayer', help='Base directory of the destination layer to write the bbappend to', type=layer)
+ parser.add_argument('target', help='Target recipe/provide to append')
+ parser.set_defaults(func=newappend, parserecipes=True)
diff --git a/scripts/lib/recipetool/setvar.py b/scripts/lib/recipetool/setvar.py
new file mode 100644
index 0000000000..9de315a0ef
--- /dev/null
+++ b/scripts/lib/recipetool/setvar.py
@@ -0,0 +1,75 @@
+# Recipe creation tool - set variable plugin
+#
+# Copyright (C) 2015 Intel Corporation
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import sys
+import os
+import argparse
+import glob
+import fnmatch
+import re
+import logging
+import scriptutils
+
+logger = logging.getLogger('recipetool')
+
+tinfoil = None
+plugins = None
+
+def tinfoil_init(instance):
+ global tinfoil
+ tinfoil = instance
+
+def setvar(args):
+ import oe.recipeutils
+
+ if args.delete:
+ if args.value:
+ logger.error('-D/--delete and specifying a value are mutually exclusive')
+ return 1
+ value = None
+ else:
+ if args.value is None:
+ logger.error('You must specify a value if not using -D/--delete')
+ return 1
+ value = args.value
+ varvalues = {args.varname: value}
+
+ if args.recipe_only:
+ patches = [oe.recipeutils.patch_recipe_file(args.recipefile, varvalues, patch=args.patch)]
+ else:
+ rd = tinfoil.parse_recipe_file(args.recipefile, False)
+ if not rd:
+ return 1
+ patches = oe.recipeutils.patch_recipe(rd, args.recipefile, varvalues, patch=args.patch)
+ if args.patch:
+ for patch in patches:
+ for line in patch:
+ sys.stdout.write(line)
+ return 0
+
+
+def register_commands(subparsers):
+ parser_setvar = subparsers.add_parser('setvar',
+ help='Set a variable within a recipe',
+ description='Adds/updates the value a variable is set to in a recipe')
+ parser_setvar.add_argument('recipefile', help='Recipe file to update')
+ parser_setvar.add_argument('varname', help='Variable name to set')
+ parser_setvar.add_argument('value', nargs='?', help='New value to set the variable to')
+ parser_setvar.add_argument('--recipe-only', '-r', help='Do not set variable in any include file if present', action='store_true')
+ parser_setvar.add_argument('--patch', '-p', help='Create a patch to make the change instead of modifying the recipe', action='store_true')
+ parser_setvar.add_argument('--delete', '-D', help='Delete the specified value instead of setting it', action='store_true')
+ parser_setvar.set_defaults(func=setvar)
diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py
index fdf4b5d55d..4ccbe5c108 100644
--- a/scripts/lib/scriptutils.py
+++ b/scripts/lib/scriptutils.py
@@ -19,10 +19,12 @@ import sys
import os
import logging
import glob
+import argparse
+import subprocess
-def logger_create(name):
+def logger_create(name, stream=None):
logger = logging.getLogger(name)
- loggerhandler = logging.StreamHandler()
+ loggerhandler = logging.StreamHandler(stream=stream)
loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
logger.addHandler(loggerhandler)
logger.setLevel(logging.INFO)
@@ -50,14 +52,18 @@ def load_plugins(logger, plugins, pluginpath):
if fp:
fp.close()
+ def plugin_name(filename):
+ return os.path.splitext(os.path.basename(filename))[0]
+
+ known_plugins = [plugin_name(p.__name__) for p in plugins]
logger.debug('Loading plugins from %s...' % pluginpath)
for fn in glob.glob(os.path.join(pluginpath, '*.py')):
- name = os.path.splitext(os.path.basename(fn))[0]
- if name != '__init__':
+ name = plugin_name(fn)
+ if name != '__init__' and name not in known_plugins:
plugin = load_plugin(name)
if hasattr(plugin, 'plugin_init'):
plugin.plugin_init(plugins)
- plugins.append(plugin)
+ plugins.append(plugin)
def git_convert_standalone_clone(repodir):
"""If specified directory is a git repository, ensure it's a standalone clone"""
@@ -69,3 +75,61 @@ def git_convert_standalone_clone(repodir):
# of the contents is shared
bb.process.run('git repack -a', cwd=repodir)
os.remove(alternatesfile)
+
+def fetch_uri(d, uri, destdir, srcrev=None):
+ """Fetch a URI to a local directory"""
+ import bb.data
+ bb.utils.mkdirhier(destdir)
+ localdata = bb.data.createCopy(d)
+ localdata.setVar('BB_STRICT_CHECKSUM', '')
+ localdata.setVar('SRCREV', srcrev)
+ ret = (None, None)
+ olddir = os.getcwd()
+ try:
+ fetcher = bb.fetch2.Fetch([uri], localdata)
+ for u in fetcher.ud:
+ ud = fetcher.ud[u]
+ ud.ignore_checksums = True
+ fetcher.download()
+ for u in fetcher.ud:
+ ud = fetcher.ud[u]
+ if ud.localpath.rstrip(os.sep) == localdata.getVar('DL_DIR').rstrip(os.sep):
+ raise Exception('Local path is download directory - please check that the URI "%s" is correct' % uri)
+ fetcher.unpack(destdir)
+ for u in fetcher.ud:
+ ud = fetcher.ud[u]
+ if ud.method.recommends_checksum(ud):
+ md5value = bb.utils.md5_file(ud.localpath)
+ sha256value = bb.utils.sha256_file(ud.localpath)
+ ret = (md5value, sha256value)
+ finally:
+ os.chdir(olddir)
+ return ret
+
+def run_editor(fn):
+ if isinstance(fn, str):
+ params = '"%s"' % fn
+ else:
+ params = ''
+ for fnitem in fn:
+ params += ' "%s"' % fnitem
+
+ editor = os.getenv('VISUAL', os.getenv('EDITOR', 'vi'))
+ try:
+ return subprocess.check_call('%s %s' % (editor, params), shell=True)
+ except OSError as exc:
+ logger.error("Execution of editor '%s' failed: %s", editor, exc)
+ return 1
+
+def is_src_url(param):
+ """
+ Check if a parameter is a URL and return True if so
+ NOTE: be careful about changing this as it will influence how devtool/recipetool command line handling works
+ """
+ if not param:
+ return False
+ elif '://' in param:
+ return True
+ elif param.startswith('git@') or ('@' in param and param.endswith('.git')):
+ return True
+ return False
diff --git a/scripts/lib/wic/3rdparty/pykickstart/base.py b/scripts/lib/wic/3rdparty/pykickstart/base.py
deleted file mode 100644
index e6c8f56f9d..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/base.py
+++ /dev/null
@@ -1,466 +0,0 @@
-#
-# Chris Lumens <clumens@redhat.com>
-#
-# Copyright 2006, 2007, 2008 Red Hat, Inc.
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
-# trademarks that are incorporated in the source code or documentation are not
-# subject to the GNU General Public License and may only be used or replicated
-# with the express permission of Red Hat, Inc.
-#
-"""
-Base classes for creating commands and syntax version object.
-
-This module exports several important base classes:
-
- BaseData - The base abstract class for all data objects. Data objects
- are contained within a BaseHandler object.
-
- BaseHandler - The base abstract class from which versioned kickstart
- handler are derived. Subclasses of BaseHandler hold
- BaseData and KickstartCommand objects.
-
- DeprecatedCommand - An abstract subclass of KickstartCommand that should
- be further subclassed by users of this module. When
- a subclass is used, a warning message will be
- printed.
-
- KickstartCommand - The base abstract class for all kickstart commands.
- Command objects are contained within a BaseHandler
- object.
-"""
-import gettext
-gettext.textdomain("pykickstart")
-_ = lambda x: gettext.ldgettext("pykickstart", x)
-
-import types
-import warnings
-from pykickstart.errors import *
-from pykickstart.ko import *
-from pykickstart.parser import Packages
-from pykickstart.version import versionToString
-
-###
-### COMMANDS
-###
-class KickstartCommand(KickstartObject):
- """The base class for all kickstart commands. This is an abstract class."""
- removedKeywords = []
- removedAttrs = []
-
- def __init__(self, writePriority=0, *args, **kwargs):
- """Create a new KickstartCommand instance. This method must be
- provided by all subclasses, but subclasses must call
- KickstartCommand.__init__ first. Instance attributes:
-
- currentCmd -- The name of the command in the input file that
- caused this handler to be run.
- currentLine -- The current unprocessed line from the input file
- that caused this handler to be run.
- handler -- A reference to the BaseHandler subclass this
- command is contained withing. This is needed to
- allow referencing of Data objects.
- lineno -- The current line number in the input file.
- writePriority -- An integer specifying when this command should be
- printed when iterating over all commands' __str__
- methods. The higher the number, the later this
- command will be written. All commands with the
- same priority will be written alphabetically.
- """
-
- # We don't want people using this class by itself.
- if self.__class__ is KickstartCommand:
- raise TypeError, "KickstartCommand is an abstract class."
-
- KickstartObject.__init__(self, *args, **kwargs)
-
- self.writePriority = writePriority
-
- # These will be set by the dispatcher.
- self.currentCmd = ""
- self.currentLine = ""
- self.handler = None
- self.lineno = 0
-
- # If a subclass provides a removedKeywords list, remove all the
- # members from the kwargs list before we start processing it. This
- # ensures that subclasses don't continue to recognize arguments that
- # were removed.
- for arg in filter(kwargs.has_key, self.removedKeywords):
- kwargs.pop(arg)
-
- def __call__(self, *args, **kwargs):
- """Set multiple attributes on a subclass of KickstartCommand at once
- via keyword arguments. Valid attributes are anything specified in
- a subclass, but unknown attributes will be ignored.
- """
- for (key, val) in kwargs.items():
- # Ignore setting attributes that were removed in a subclass, as
- # if they were unknown attributes.
- if key in self.removedAttrs:
- continue
-
- if hasattr(self, key):
- setattr(self, key, val)
-
- def __str__(self):
- """Return a string formatted for output to a kickstart file. This
- method must be provided by all subclasses.
- """
- return KickstartObject.__str__(self)
-
- def parse(self, args):
- """Parse the list of args and set data on the KickstartCommand object.
- This method must be provided by all subclasses.
- """
- raise TypeError, "parse() not implemented for KickstartCommand"
-
- def apply(self, instroot="/"):
- """Write out the configuration related to the KickstartCommand object.
- Subclasses which do not provide this method will not have their
- configuration written out.
- """
- return
-
- def dataList(self):
- """For commands that can occur multiple times in a single kickstart
- file (like network, part, etc.), return the list that we should
- append more data objects to.
- """
- return None
-
- def deleteRemovedAttrs(self):
- """Remove all attributes from self that are given in the removedAttrs
- list. This method should be called from __init__ in a subclass,
- but only after the superclass's __init__ method has been called.
- """
- for attr in filter(lambda k: hasattr(self, k), self.removedAttrs):
- delattr(self, attr)
-
- # Set the contents of the opts object (an instance of optparse.Values
- # returned by parse_args) as attributes on the KickstartCommand object.
- # It's useful to call this from KickstartCommand subclasses after parsing
- # the arguments.
- def _setToSelf(self, optParser, opts):
- self._setToObj(optParser, opts, self)
-
- # Sets the contents of the opts object (an instance of optparse.Values
- # returned by parse_args) as attributes on the provided object obj. It's
- # useful to call this from KickstartCommand subclasses that handle lists
- # of objects (like partitions, network devices, etc.) and need to populate
- # a Data object.
- def _setToObj(self, optParser, opts, obj):
- for key in filter (lambda k: getattr(opts, k) != None, optParser.keys()):
- setattr(obj, key, getattr(opts, key))
-
-class DeprecatedCommand(KickstartCommand):
- """Specify that a command is deprecated and no longer has any function.
- Any command that is deprecated should be subclassed from this class,
- only specifying an __init__ method that calls the superclass's __init__.
- This is an abstract class.
- """
- def __init__(self, writePriority=None, *args, **kwargs):
- # We don't want people using this class by itself.
- if self.__class__ is KickstartCommand:
- raise TypeError, "DeprecatedCommand is an abstract class."
-
- # Create a new DeprecatedCommand instance.
- KickstartCommand.__init__(self, writePriority, *args, **kwargs)
-
- def __str__(self):
- """Placeholder since DeprecatedCommands don't work anymore."""
- return ""
-
- def parse(self, args):
- """Print a warning message if the command is seen in the input file."""
- mapping = {"lineno": self.lineno, "cmd": self.currentCmd}
- warnings.warn(_("Ignoring deprecated command on line %(lineno)s: The %(cmd)s command has been deprecated and no longer has any effect. It may be removed from future releases, which will result in a fatal error from kickstart. Please modify your kickstart file to remove this command.") % mapping, DeprecationWarning)
-
-
-###
-### HANDLERS
-###
-class BaseHandler(KickstartObject):
- """Each version of kickstart syntax is provided by a subclass of this
- class. These subclasses are what users will interact with for parsing,
- extracting data, and writing out kickstart files. This is an abstract
- class.
-
- version -- The version this syntax handler supports. This is set by
- a class attribute of a BaseHandler subclass and is used to
- set up the command dict. It is for read-only use.
- """
- version = None
-
- def __init__(self, mapping=None, dataMapping=None, commandUpdates=None,
- dataUpdates=None, *args, **kwargs):
- """Create a new BaseHandler instance. This method must be provided by
- all subclasses, but subclasses must call BaseHandler.__init__ first.
-
- mapping -- A custom map from command strings to classes,
- useful when creating your own handler with
- special command objects. It is otherwise unused
- and rarely needed. If you give this argument,
- the mapping takes the place of the default one
- and so must include all commands you want
- recognized.
- dataMapping -- This is the same as mapping, but for data
- objects. All the same comments apply.
- commandUpdates -- This is similar to mapping, but does not take
- the place of the defaults entirely. Instead,
- this mapping is applied after the defaults and
- updates it with just the commands you want to
- modify.
- dataUpdates -- This is the same as commandUpdates, but for
- data objects.
-
-
- Instance attributes:
-
- commands -- A mapping from a string command to a KickstartCommand
- subclass object that handles it. Multiple strings can
- map to the same object, but only one instance of the
- command object should ever exist. Most users should
- never have to deal with this directly, as it is
- manipulated internally and called through dispatcher.
- currentLine -- The current unprocessed line from the input file
- that caused this handler to be run.
- packages -- An instance of pykickstart.parser.Packages which
- describes the packages section of the input file.
- platform -- A string describing the hardware platform, which is
- needed only by system-config-kickstart.
- scripts -- A list of pykickstart.parser.Script instances, which is
- populated by KickstartParser.addScript and describes the
- %pre/%post/%traceback script section of the input file.
- """
-
- # We don't want people using this class by itself.
- if self.__class__ is BaseHandler:
- raise TypeError, "BaseHandler is an abstract class."
-
- KickstartObject.__init__(self, *args, **kwargs)
-
- # This isn't really a good place for these, but it's better than
- # everything else I can think of.
- self.scripts = []
- self.packages = Packages()
- self.platform = ""
-
- # These will be set by the dispatcher.
- self.commands = {}
- self.currentLine = 0
-
- # A dict keyed by an integer priority number, with each value being a
- # list of KickstartCommand subclasses. This dict is maintained by
- # registerCommand and used in __str__. No one else should be touching
- # it.
- self._writeOrder = {}
-
- self._registerCommands(mapping, dataMapping, commandUpdates, dataUpdates)
-
- def __str__(self):
- """Return a string formatted for output to a kickstart file."""
- retval = ""
-
- if self.platform != "":
- retval += "#platform=%s\n" % self.platform
-
- retval += "#version=%s\n" % versionToString(self.version)
-
- lst = self._writeOrder.keys()
- lst.sort()
-
- for prio in lst:
- for obj in self._writeOrder[prio]:
- retval += obj.__str__()
-
- for script in self.scripts:
- retval += script.__str__()
-
- retval += self.packages.__str__()
-
- return retval
-
- def _insertSorted(self, lst, obj):
- length = len(lst)
- i = 0
-
- while i < length:
- # If the two classes have the same name, it's because we are
- # overriding an existing class with one from a later kickstart
- # version, so remove the old one in favor of the new one.
- if obj.__class__.__name__ > lst[i].__class__.__name__:
- i += 1
- elif obj.__class__.__name__ == lst[i].__class__.__name__:
- lst[i] = obj
- return
- elif obj.__class__.__name__ < lst[i].__class__.__name__:
- break
-
- if i >= length:
- lst.append(obj)
- else:
- lst.insert(i, obj)
-
- def _setCommand(self, cmdObj):
- # Add an attribute on this version object. We need this to provide a
- # way for clients to access the command objects. We also need to strip
- # off the version part from the front of the name.
- if cmdObj.__class__.__name__.find("_") != -1:
- name = unicode(cmdObj.__class__.__name__.split("_", 1)[1])
- else:
- name = unicode(cmdObj.__class__.__name__).lower()
-
- setattr(self, name.lower(), cmdObj)
-
- # Also, add the object into the _writeOrder dict in the right place.
- if cmdObj.writePriority is not None:
- if self._writeOrder.has_key(cmdObj.writePriority):
- self._insertSorted(self._writeOrder[cmdObj.writePriority], cmdObj)
- else:
- self._writeOrder[cmdObj.writePriority] = [cmdObj]
-
- def _registerCommands(self, mapping=None, dataMapping=None, commandUpdates=None,
- dataUpdates=None):
- if mapping == {} or mapping == None:
- from pykickstart.handlers.control import commandMap
- cMap = commandMap[self.version]
- else:
- cMap = mapping
-
- if dataMapping == {} or dataMapping == None:
- from pykickstart.handlers.control import dataMap
- dMap = dataMap[self.version]
- else:
- dMap = dataMapping
-
- if type(commandUpdates) == types.DictType:
- cMap.update(commandUpdates)
-
- if type(dataUpdates) == types.DictType:
- dMap.update(dataUpdates)
-
- for (cmdName, cmdClass) in cMap.iteritems():
- # First make sure we haven't instantiated this command handler
- # already. If we have, we just need to make another mapping to
- # it in self.commands.
- cmdObj = None
-
- for (key, val) in self.commands.iteritems():
- if val.__class__.__name__ == cmdClass.__name__:
- cmdObj = val
- break
-
- # If we didn't find an instance in self.commands, create one now.
- if cmdObj == None:
- cmdObj = cmdClass()
- self._setCommand(cmdObj)
-
- # Finally, add the mapping to the commands dict.
- self.commands[cmdName] = cmdObj
- self.commands[cmdName].handler = self
-
- # We also need to create attributes for the various data objects.
- # No checks here because dMap is a bijection. At least, that's what
- # the comment says. Hope no one screws that up.
- for (dataName, dataClass) in dMap.iteritems():
- setattr(self, dataName, dataClass)
-
- def dispatcher(self, args, lineno):
- """Call the appropriate KickstartCommand handler for the current line
- in the kickstart file. A handler for the current command should
- be registered, though a handler of None is not an error. Returns
- the data object returned by KickstartCommand.parse.
-
- args -- A list of arguments to the current command
- lineno -- The line number in the file, for error reporting
- """
- cmd = args[0]
-
- if not self.commands.has_key(cmd):
- raise KickstartParseError, formatErrorMsg(lineno, msg=_("Unknown command: %s" % cmd))
- elif self.commands[cmd] != None:
- self.commands[cmd].currentCmd = cmd
- self.commands[cmd].currentLine = self.currentLine
- self.commands[cmd].lineno = lineno
-
- # The parser returns the data object that was modified. This could
- # be a BaseData subclass that should be put into a list, or it
- # could be the command handler object itself.
- obj = self.commands[cmd].parse(args[1:])
- lst = self.commands[cmd].dataList()
- if lst is not None:
- lst.append(obj)
-
- return obj
-
- def maskAllExcept(self, lst):
- """Set all entries in the commands dict to None, except the ones in
- the lst. All other commands will not be processed.
- """
- self._writeOrder = {}
-
- for (key, val) in self.commands.iteritems():
- if not key in lst:
- self.commands[key] = None
-
- def hasCommand(self, cmd):
- """Return true if there is a handler for the string cmd."""
- return hasattr(self, cmd)
-
-
-###
-### DATA
-###
-class BaseData(KickstartObject):
- """The base class for all data objects. This is an abstract class."""
- removedKeywords = []
- removedAttrs = []
-
- def __init__(self, *args, **kwargs):
- """Create a new BaseData instance.
-
- lineno -- Line number in the ks-file where this object was defined
- """
-
- # We don't want people using this class by itself.
- if self.__class__ is BaseData:
- raise TypeError, "BaseData is an abstract class."
-
- KickstartObject.__init__(self, *args, **kwargs)
- self.lineno = 0
-
- def __str__(self):
- """Return a string formatted for output to a kickstart file."""
- return ""
-
- def __call__(self, *args, **kwargs):
- """Set multiple attributes on a subclass of BaseData at once via
- keyword arguments. Valid attributes are anything specified in a
- subclass, but unknown attributes will be ignored.
- """
- for (key, val) in kwargs.items():
- # Ignore setting attributes that were removed in a subclass, as
- # if they were unknown attributes.
- if key in self.removedAttrs:
- continue
-
- if hasattr(self, key):
- setattr(self, key, val)
-
- def deleteRemovedAttrs(self):
- """Remove all attributes from self that are given in the removedAttrs
- list. This method should be called from __init__ in a subclass,
- but only after the superclass's __init__ method has been called.
- """
- for attr in filter(lambda k: hasattr(self, k), self.removedAttrs):
- delattr(self, attr)
diff --git a/scripts/lib/wic/3rdparty/pykickstart/commands/__init__.py b/scripts/lib/wic/3rdparty/pykickstart/commands/__init__.py
deleted file mode 100644
index 2d94550935..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/commands/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Chris Lumens <clumens@redhat.com>
-#
-# Copyright 2009 Red Hat, Inc.
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
-# trademarks that are incorporated in the source code or documentation are not
-# subject to the GNU General Public License and may only be used or replicated
-# with the express permission of Red Hat, Inc.
-#
-import bootloader, partition
diff --git a/scripts/lib/wic/3rdparty/pykickstart/commands/bootloader.py b/scripts/lib/wic/3rdparty/pykickstart/commands/bootloader.py
deleted file mode 100644
index c2b552f689..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/commands/bootloader.py
+++ /dev/null
@@ -1,216 +0,0 @@
-#
-# Chris Lumens <clumens@redhat.com>
-#
-# Copyright 2007 Red Hat, Inc.
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
-# trademarks that are incorporated in the source code or documentation are not
-# subject to the GNU General Public License and may only be used or replicated
-# with the express permission of Red Hat, Inc.
-#
-from pykickstart.base import *
-from pykickstart.options import *
-
-class FC3_Bootloader(KickstartCommand):
- removedKeywords = KickstartCommand.removedKeywords
- removedAttrs = KickstartCommand.removedAttrs
-
- def __init__(self, writePriority=10, *args, **kwargs):
- KickstartCommand.__init__(self, writePriority, *args, **kwargs)
- self.op = self._getParser()
-
- self.driveorder = kwargs.get("driveorder", [])
- self.appendLine = kwargs.get("appendLine", "")
- self.forceLBA = kwargs.get("forceLBA", False)
- self.linear = kwargs.get("linear", True)
- self.location = kwargs.get("location", "")
- self.md5pass = kwargs.get("md5pass", "")
- self.password = kwargs.get("password", "")
- self.upgrade = kwargs.get("upgrade", False)
- self.useLilo = kwargs.get("useLilo", False)
-
- self.deleteRemovedAttrs()
-
- def _getArgsAsStr(self):
- retval = ""
-
- if self.appendLine != "":
- retval += " --append=\"%s\"" % self.appendLine
- if self.linear:
- retval += " --linear"
- if self.location:
- retval += " --location=%s" % self.location
- if hasattr(self, "forceLBA") and self.forceLBA:
- retval += " --lba32"
- if self.password != "":
- retval += " --password=\"%s\"" % self.password
- if self.md5pass != "":
- retval += " --md5pass=\"%s\"" % self.md5pass
- if self.upgrade:
- retval += " --upgrade"
- if self.useLilo:
- retval += " --useLilo"
- if len(self.driveorder) > 0:
- retval += " --driveorder=\"%s\"" % ",".join(self.driveorder)
-
- return retval
-
- def __str__(self):
- retval = KickstartCommand.__str__(self)
-
- if self.location != "":
- retval += "# System bootloader configuration\nbootloader"
- retval += self._getArgsAsStr() + "\n"
-
- return retval
-
- def _getParser(self):
- def driveorder_cb (option, opt_str, value, parser):
- for d in value.split(','):
- parser.values.ensure_value(option.dest, []).append(d)
-
- op = KSOptionParser()
- op.add_option("--append", dest="appendLine")
- op.add_option("--linear", dest="linear", action="store_true",
- default=True)
- op.add_option("--nolinear", dest="linear", action="store_false")
- op.add_option("--location", dest="location", type="choice",
- default="mbr",
- choices=["mbr", "partition", "none", "boot"])
- op.add_option("--lba32", dest="forceLBA", action="store_true",
- default=False)
- op.add_option("--password", dest="password", default="")
- op.add_option("--md5pass", dest="md5pass", default="")
- op.add_option("--upgrade", dest="upgrade", action="store_true",
- default=False)
- op.add_option("--useLilo", dest="useLilo", action="store_true",
- default=False)
- op.add_option("--driveorder", dest="driveorder", action="callback",
- callback=driveorder_cb, nargs=1, type="string")
- return op
-
- def parse(self, args):
- (opts, extra) = self.op.parse_args(args=args, lineno=self.lineno)
- self._setToSelf(self.op, opts)
-
- if self.currentCmd == "lilo":
- self.useLilo = True
-
- return self
-
-class FC4_Bootloader(FC3_Bootloader):
- removedKeywords = FC3_Bootloader.removedKeywords + ["linear", "useLilo"]
- removedAttrs = FC3_Bootloader.removedAttrs + ["linear", "useLilo"]
-
- def __init__(self, writePriority=10, *args, **kwargs):
- FC3_Bootloader.__init__(self, writePriority, *args, **kwargs)
-
- def _getArgsAsStr(self):
- retval = ""
- if self.appendLine != "":
- retval += " --append=\"%s\"" % self.appendLine
- if self.location:
- retval += " --location=%s" % self.location
- if hasattr(self, "forceLBA") and self.forceLBA:
- retval += " --lba32"
- if self.password != "":
- retval += " --password=\"%s\"" % self.password
- if self.md5pass != "":
- retval += " --md5pass=\"%s\"" % self.md5pass
- if self.upgrade:
- retval += " --upgrade"
- if len(self.driveorder) > 0:
- retval += " --driveorder=\"%s\"" % ",".join(self.driveorder)
- return retval
-
- def _getParser(self):
- op = FC3_Bootloader._getParser(self)
- op.remove_option("--linear")
- op.remove_option("--nolinear")
- op.remove_option("--useLilo")
- return op
-
- def parse(self, args):
- (opts, extra) = self.op.parse_args(args=args, lineno=self.lineno)
- self._setToSelf(self.op, opts)
- return self
-
-class F8_Bootloader(FC4_Bootloader):
- removedKeywords = FC4_Bootloader.removedKeywords
- removedAttrs = FC4_Bootloader.removedAttrs
-
- def __init__(self, writePriority=10, *args, **kwargs):
- FC4_Bootloader.__init__(self, writePriority, *args, **kwargs)
-
- self.timeout = kwargs.get("timeout", None)
- self.default = kwargs.get("default", "")
-
- def _getArgsAsStr(self):
- ret = FC4_Bootloader._getArgsAsStr(self)
-
- if self.timeout is not None:
- ret += " --timeout=%d" %(self.timeout,)
- if self.default:
- ret += " --default=%s" %(self.default,)
-
- return ret
-
- def _getParser(self):
- op = FC4_Bootloader._getParser(self)
- op.add_option("--timeout", dest="timeout", type="int")
- op.add_option("--default", dest="default")
- return op
-
-class F12_Bootloader(F8_Bootloader):
- removedKeywords = F8_Bootloader.removedKeywords
- removedAttrs = F8_Bootloader.removedAttrs
-
- def _getParser(self):
- op = F8_Bootloader._getParser(self)
- op.add_option("--lba32", dest="forceLBA", deprecated=1, action="store_true")
- return op
-
-class F14_Bootloader(F12_Bootloader):
- removedKeywords = F12_Bootloader.removedKeywords + ["forceLBA"]
- removedAttrs = F12_Bootloader.removedKeywords + ["forceLBA"]
-
- def _getParser(self):
- op = F12_Bootloader._getParser(self)
- op.remove_option("--lba32")
- return op
-
-class F15_Bootloader(F14_Bootloader):
- removedKeywords = F14_Bootloader.removedKeywords
- removedAttrs = F14_Bootloader.removedAttrs
-
- def __init__(self, writePriority=10, *args, **kwargs):
- F14_Bootloader.__init__(self, writePriority, *args, **kwargs)
-
- self.isCrypted = kwargs.get("isCrypted", False)
-
- def _getArgsAsStr(self):
- ret = F14_Bootloader._getArgsAsStr(self)
-
- if self.isCrypted:
- ret += " --iscrypted"
-
- return ret
-
- def _getParser(self):
- def password_cb(option, opt_str, value, parser):
- parser.values.isCrypted = True
- parser.values.password = value
-
- op = F14_Bootloader._getParser(self)
- op.add_option("--iscrypted", dest="isCrypted", action="store_true", default=False)
- op.add_option("--md5pass", action="callback", callback=password_cb, nargs=1, type="string")
- return op
diff --git a/scripts/lib/wic/3rdparty/pykickstart/commands/partition.py b/scripts/lib/wic/3rdparty/pykickstart/commands/partition.py
deleted file mode 100644
index b564b1a7ab..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/commands/partition.py
+++ /dev/null
@@ -1,314 +0,0 @@
-#
-# Chris Lumens <clumens@redhat.com>
-#
-# Copyright 2005, 2006, 2007, 2008 Red Hat, Inc.
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
-# trademarks that are incorporated in the source code or documentation are not
-# subject to the GNU General Public License and may only be used or replicated
-# with the express permission of Red Hat, Inc.
-#
-from pykickstart.base import *
-from pykickstart.errors import *
-from pykickstart.options import *
-
-import gettext
-import warnings
-_ = lambda x: gettext.ldgettext("pykickstart", x)
-
-class FC3_PartData(BaseData):
- removedKeywords = BaseData.removedKeywords
- removedAttrs = BaseData.removedAttrs
-
- def __init__(self, *args, **kwargs):
- BaseData.__init__(self, *args, **kwargs)
- self.active = kwargs.get("active", False)
- self.primOnly = kwargs.get("primOnly", False)
- self.end = kwargs.get("end", 0)
- self.fstype = kwargs.get("fstype", "")
- self.grow = kwargs.get("grow", False)
- self.maxSizeMB = kwargs.get("maxSizeMB", 0)
- self.format = kwargs.get("format", True)
- self.onbiosdisk = kwargs.get("onbiosdisk", "")
- self.disk = kwargs.get("disk", "")
- self.onPart = kwargs.get("onPart", "")
- self.recommended = kwargs.get("recommended", False)
- self.size = kwargs.get("size", None)
- self.start = kwargs.get("start", 0)
- self.mountpoint = kwargs.get("mountpoint", "")
-
- def __eq__(self, y):
- if self.mountpoint:
- return self.mountpoint == y.mountpoint
- else:
- return False
-
- def _getArgsAsStr(self):
- retval = ""
-
- if self.active:
- retval += " --active"
- if self.primOnly:
- retval += " --asprimary"
- if hasattr(self, "end") and self.end != 0:
- retval += " --end=%s" % self.end
- if self.fstype != "":
- retval += " --fstype=\"%s\"" % self.fstype
- if self.grow:
- retval += " --grow"
- if self.maxSizeMB > 0:
- retval += " --maxsize=%d" % self.maxSizeMB
- if not self.format:
- retval += " --noformat"
- if self.onbiosdisk != "":
- retval += " --onbiosdisk=%s" % self.onbiosdisk
- if self.disk != "":
- retval += " --ondisk=%s" % self.disk
- if self.onPart != "":
- retval += " --onpart=%s" % self.onPart
- if self.recommended:
- retval += " --recommended"
- if self.size and self.size != 0:
- retval += " --size=%sk" % self.size
- if hasattr(self, "start") and self.start != 0:
- retval += " --start=%s" % self.start
-
- return retval
-
- def __str__(self):
- retval = BaseData.__str__(self)
- if self.mountpoint:
- mountpoint_str = "%s" % self.mountpoint
- else:
- mountpoint_str = "(No mount point)"
- retval += "part %s%s\n" % (mountpoint_str, self._getArgsAsStr())
- return retval
-
-class FC4_PartData(FC3_PartData):
- removedKeywords = FC3_PartData.removedKeywords
- removedAttrs = FC3_PartData.removedAttrs
-
- def __init__(self, *args, **kwargs):
- FC3_PartData.__init__(self, *args, **kwargs)
- self.bytesPerInode = kwargs.get("bytesPerInode", 4096)
- self.fsopts = kwargs.get("fsopts", "")
- self.label = kwargs.get("label", "")
-
- def _getArgsAsStr(self):
- retval = FC3_PartData._getArgsAsStr(self)
-
- if hasattr(self, "bytesPerInode") and self.bytesPerInode != 0:
- retval += " --bytes-per-inode=%d" % self.bytesPerInode
- if self.fsopts != "":
- retval += " --fsoptions=\"%s\"" % self.fsopts
- if self.label != "":
- retval += " --label=%s" % self.label
-
- return retval
-
-class F9_PartData(FC4_PartData):
- removedKeywords = FC4_PartData.removedKeywords + ["bytesPerInode"]
- removedAttrs = FC4_PartData.removedAttrs + ["bytesPerInode"]
-
- def __init__(self, *args, **kwargs):
- FC4_PartData.__init__(self, *args, **kwargs)
- self.deleteRemovedAttrs()
-
- self.fsopts = kwargs.get("fsopts", "")
- self.label = kwargs.get("label", "")
- self.fsprofile = kwargs.get("fsprofile", "")
- self.encrypted = kwargs.get("encrypted", False)
- self.passphrase = kwargs.get("passphrase", "")
-
- def _getArgsAsStr(self):
- retval = FC4_PartData._getArgsAsStr(self)
-
- if self.fsprofile != "":
- retval += " --fsprofile=\"%s\"" % self.fsprofile
- if self.encrypted:
- retval += " --encrypted"
-
- if self.passphrase != "":
- retval += " --passphrase=\"%s\"" % self.passphrase
-
- return retval
-
-class F11_PartData(F9_PartData):
- removedKeywords = F9_PartData.removedKeywords + ["start", "end"]
- removedAttrs = F9_PartData.removedAttrs + ["start", "end"]
-
-class F12_PartData(F11_PartData):
- removedKeywords = F11_PartData.removedKeywords
- removedAttrs = F11_PartData.removedAttrs
-
- def __init__(self, *args, **kwargs):
- F11_PartData.__init__(self, *args, **kwargs)
-
- self.escrowcert = kwargs.get("escrowcert", "")
- self.backuppassphrase = kwargs.get("backuppassphrase", False)
-
- def _getArgsAsStr(self):
- retval = F11_PartData._getArgsAsStr(self)
-
- if self.encrypted and self.escrowcert != "":
- retval += " --escrowcert=\"%s\"" % self.escrowcert
-
- if self.backuppassphrase:
- retval += " --backuppassphrase"
-
- return retval
-
-F14_PartData = F12_PartData
-
-class FC3_Partition(KickstartCommand):
- removedKeywords = KickstartCommand.removedKeywords
- removedAttrs = KickstartCommand.removedAttrs
-
- def __init__(self, writePriority=130, *args, **kwargs):
- KickstartCommand.__init__(self, writePriority, *args, **kwargs)
- self.op = self._getParser()
-
- self.partitions = kwargs.get("partitions", [])
-
- def __str__(self):
- retval = ""
-
- for part in self.partitions:
- retval += part.__str__()
-
- if retval != "":
- return "# Disk partitioning information\n" + retval
- else:
- return ""
-
- def _getParser(self):
- def part_cb (option, opt_str, value, parser):
- if value.startswith("/dev/"):
- parser.values.ensure_value(option.dest, value[5:])
- else:
- parser.values.ensure_value(option.dest, value)
-
- op = KSOptionParser()
- op.add_option("--active", dest="active", action="store_true",
- default=False)
- op.add_option("--asprimary", dest="primOnly", action="store_true",
- default=False)
- op.add_option("--end", dest="end", action="store", type="int",
- nargs=1)
- op.add_option("--fstype", "--type", dest="fstype")
- op.add_option("--grow", dest="grow", action="store_true", default=False)
- op.add_option("--maxsize", dest="maxSizeMB", action="store", type="int",
- nargs=1)
- op.add_option("--noformat", dest="format", action="store_false",
- default=True)
- op.add_option("--onbiosdisk", dest="onbiosdisk")
- op.add_option("--ondisk", "--ondrive", dest="disk")
- op.add_option("--onpart", "--usepart", dest="onPart", action="callback",
- callback=part_cb, nargs=1, type="string")
- op.add_option("--recommended", dest="recommended", action="store_true",
- default=False)
- op.add_option("--size", dest="size", action="store", type="size",
- nargs=1)
- op.add_option("--start", dest="start", action="store", type="int",
- nargs=1)
- return op
-
- def parse(self, args):
- (opts, extra) = self.op.parse_args(args=args, lineno=self.lineno)
-
- pd = self.handler.PartData()
- self._setToObj(self.op, opts, pd)
- pd.lineno = self.lineno
- if extra:
- pd.mountpoint = extra[0]
- if pd in self.dataList():
- warnings.warn(_("A partition with the mountpoint %s has already been defined.") % pd.mountpoint)
- else:
- pd.mountpoint = None
-
- return pd
-
- def dataList(self):
- return self.partitions
-
-class FC4_Partition(FC3_Partition):
- removedKeywords = FC3_Partition.removedKeywords
- removedAttrs = FC3_Partition.removedAttrs
-
- def __init__(self, writePriority=130, *args, **kwargs):
- FC3_Partition.__init__(self, writePriority, *args, **kwargs)
-
- def part_cb (option, opt_str, value, parser):
- if value.startswith("/dev/"):
- parser.values.ensure_value(option.dest, value[5:])
- else:
- parser.values.ensure_value(option.dest, value)
-
- def _getParser(self):
- op = FC3_Partition._getParser(self)
- op.add_option("--bytes-per-inode", dest="bytesPerInode", action="store",
- type="int", nargs=1)
- op.add_option("--fsoptions", dest="fsopts")
- op.add_option("--label", dest="label")
- return op
-
-class F9_Partition(FC4_Partition):
- removedKeywords = FC4_Partition.removedKeywords
- removedAttrs = FC4_Partition.removedAttrs
-
- def __init__(self, writePriority=130, *args, **kwargs):
- FC4_Partition.__init__(self, writePriority, *args, **kwargs)
-
- def part_cb (option, opt_str, value, parser):
- if value.startswith("/dev/"):
- parser.values.ensure_value(option.dest, value[5:])
- else:
- parser.values.ensure_value(option.dest, value)
-
- def _getParser(self):
- op = FC4_Partition._getParser(self)
- op.add_option("--bytes-per-inode", deprecated=1)
- op.add_option("--fsprofile")
- op.add_option("--encrypted", action="store_true", default=False)
- op.add_option("--passphrase")
- return op
-
-class F11_Partition(F9_Partition):
- removedKeywords = F9_Partition.removedKeywords
- removedAttrs = F9_Partition.removedAttrs
-
- def _getParser(self):
- op = F9_Partition._getParser(self)
- op.add_option("--start", deprecated=1)
- op.add_option("--end", deprecated=1)
- return op
-
-class F12_Partition(F11_Partition):
- removedKeywords = F11_Partition.removedKeywords
- removedAttrs = F11_Partition.removedAttrs
-
- def _getParser(self):
- op = F11_Partition._getParser(self)
- op.add_option("--escrowcert")
- op.add_option("--backuppassphrase", action="store_true", default=False)
- return op
-
-class F14_Partition(F12_Partition):
- removedKeywords = F12_Partition.removedKeywords
- removedAttrs = F12_Partition.removedAttrs
-
- def _getParser(self):
- op = F12_Partition._getParser(self)
- op.remove_option("--bytes-per-inode")
- op.remove_option("--start")
- op.remove_option("--end")
- return op
diff --git a/scripts/lib/wic/3rdparty/pykickstart/constants.py b/scripts/lib/wic/3rdparty/pykickstart/constants.py
deleted file mode 100644
index 5e12fc80ec..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/constants.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#
-# Chris Lumens <clumens@redhat.com>
-#
-# Copyright 2005-2007 Red Hat, Inc.
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
-# trademarks that are incorporated in the source code or documentation are not
-# subject to the GNU General Public License and may only be used or replicated
-# with the express permission of Red Hat, Inc.
-#
-CLEARPART_TYPE_LINUX = 0
-CLEARPART_TYPE_ALL = 1
-CLEARPART_TYPE_NONE = 2
-
-DISPLAY_MODE_CMDLINE = 0
-DISPLAY_MODE_GRAPHICAL = 1
-DISPLAY_MODE_TEXT = 2
-
-FIRSTBOOT_DEFAULT = 0
-FIRSTBOOT_SKIP = 1
-FIRSTBOOT_RECONFIG = 2
-
-KS_MISSING_PROMPT = 0
-KS_MISSING_IGNORE = 1
-
-SELINUX_DISABLED = 0
-SELINUX_ENFORCING = 1
-SELINUX_PERMISSIVE = 2
-
-KS_SCRIPT_PRE = 0
-KS_SCRIPT_POST = 1
-KS_SCRIPT_TRACEBACK = 2
-
-KS_WAIT = 0
-KS_REBOOT = 1
-KS_SHUTDOWN = 2
-
-KS_INSTKEY_SKIP = -99
-
-BOOTPROTO_DHCP = "dhcp"
-BOOTPROTO_BOOTP = "bootp"
-BOOTPROTO_STATIC = "static"
-BOOTPROTO_QUERY = "query"
-BOOTPROTO_IBFT = "ibft"
-
-GROUP_REQUIRED = 0
-GROUP_DEFAULT = 1
-GROUP_ALL = 2
diff --git a/scripts/lib/wic/3rdparty/pykickstart/errors.py b/scripts/lib/wic/3rdparty/pykickstart/errors.py
deleted file mode 100644
index a234d99d43..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/errors.py
+++ /dev/null
@@ -1,103 +0,0 @@
-#
-# errors.py: Kickstart error handling.
-#
-# Chris Lumens <clumens@redhat.com>
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
-# trademarks that are incorporated in the source code or documentation are not
-# subject to the GNU General Public License and may only be used or replicated
-# with the express permission of Red Hat, Inc.
-#
-"""
-Error handling classes and functions.
-
-This module exports a single function:
-
- formatErrorMsg - Properly formats an error message.
-
-It also exports several exception classes:
-
- KickstartError - A generic exception class.
-
- KickstartParseError - An exception for errors relating to parsing.
-
- KickstartValueError - An exception for errors relating to option
- processing.
-
- KickstartVersionError - An exception for errors relating to unsupported
- syntax versions.
-"""
-import gettext
-_ = lambda x: gettext.ldgettext("pykickstart", x)
-
-def formatErrorMsg(lineno, msg=""):
- """Properly format the error message msg for inclusion in an exception."""
- if msg != "":
- mapping = {"lineno": lineno, "msg": msg}
- return _("The following problem occurred on line %(lineno)s of the kickstart file:\n\n%(msg)s\n") % mapping
- else:
- return _("There was a problem reading from line %s of the kickstart file") % lineno
-
-class KickstartError(Exception):
- """A generic exception class for unspecific error conditions."""
- def __init__(self, val = ""):
- """Create a new KickstartError exception instance with the descriptive
- message val. val should be the return value of formatErrorMsg.
- """
- Exception.__init__(self)
- self.value = val
-
- def __str__ (self):
- return self.value
-
-class KickstartParseError(KickstartError):
- """An exception class for errors when processing the input file, such as
- unknown options, commands, or sections.
- """
- def __init__(self, msg):
- """Create a new KickstartParseError exception instance with the
- descriptive message val. val should be the return value of
- formatErrorMsg.
- """
- KickstartError.__init__(self, msg)
-
- def __str__(self):
- return self.value
-
-class KickstartValueError(KickstartError):
- """An exception class for errors when processing arguments to commands,
- such as too many arguments, too few arguments, or missing required
- arguments.
- """
- def __init__(self, msg):
- """Create a new KickstartValueError exception instance with the
- descriptive message val. val should be the return value of
- formatErrorMsg.
- """
- KickstartError.__init__(self, msg)
-
- def __str__ (self):
- return self.value
-
-class KickstartVersionError(KickstartError):
- """An exception class for errors related to using an incorrect version of
- kickstart syntax.
- """
- def __init__(self, msg):
- """Create a new KickstartVersionError exception instance with the
- descriptive message val. val should be the return value of
- formatErrorMsg.
- """
- KickstartError.__init__(self, msg)
-
- def __str__ (self):
- return self.value
diff --git a/scripts/lib/wic/3rdparty/pykickstart/handlers/__init__.py b/scripts/lib/wic/3rdparty/pykickstart/handlers/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/handlers/__init__.py
+++ /dev/null
diff --git a/scripts/lib/wic/3rdparty/pykickstart/handlers/control.py b/scripts/lib/wic/3rdparty/pykickstart/handlers/control.py
deleted file mode 100644
index 8dc80d1ebe..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/handlers/control.py
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Chris Lumens <clumens@redhat.com>
-#
-# Copyright 2007, 2008, 2009, 2010 Red Hat, Inc.
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
-# trademarks that are incorporated in the source code or documentation are not
-# subject to the GNU General Public License and may only be used or replicated
-# with the express permission of Red Hat, Inc.
-#
-from pykickstart.version import *
-from pykickstart.commands import *
-
-# This map is keyed on kickstart syntax version as provided by
-# pykickstart.version. Within each sub-dict is a mapping from command name
-# to the class that handles it. This is an onto mapping - that is, multiple
-# command names can map to the same class. However, the Handler will ensure
-# that only one instance of each class ever exists.
-commandMap = {
- # based on f15
- F16: {
- "bootloader": bootloader.F15_Bootloader,
- "part": partition.F14_Partition,
- "partition": partition.F14_Partition,
- },
-}
-
-# This map is keyed on kickstart syntax version as provided by
-# pykickstart.version. Within each sub-dict is a mapping from a data object
-# name to the class that provides it. This is a bijective mapping - that is,
-# each name maps to exactly one data class and all data classes have a name.
-# More than one instance of each class is allowed to exist, however.
-dataMap = {
- F16: {
- "PartData": partition.F14_PartData,
- },
-}
diff --git a/scripts/lib/wic/3rdparty/pykickstart/handlers/f16.py b/scripts/lib/wic/3rdparty/pykickstart/handlers/f16.py
deleted file mode 100644
index 3c52f8d754..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/handlers/f16.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Chris Lumens <clumens@redhat.com>
-#
-# Copyright 2011 Red Hat, Inc.
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
-# trademarks that are incorporated in the source code or documentation are not
-# subject to the GNU General Public License and may only be used or replicated
-# with the express permission of Red Hat, Inc.
-#
-from pykickstart.base import *
-from pykickstart.version import *
-
-class F16Handler(BaseHandler):
- version = F16
diff --git a/scripts/lib/wic/3rdparty/pykickstart/ko.py b/scripts/lib/wic/3rdparty/pykickstart/ko.py
deleted file mode 100644
index 1350d19c70..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/ko.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Chris Lumens <clumens@redhat.com>
-#
-# Copyright 2009 Red Hat, Inc.
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
-# trademarks that are incorporated in the source code or documentation are not
-# subject to the GNU General Public License and may only be used or replicated
-# with the express permission of Red Hat, Inc.
-#
-"""
-Base classes for internal pykickstart use.
-
-The module exports the following important classes:
-
- KickstartObject - The base class for all classes in pykickstart
-"""
-
-class KickstartObject(object):
- """The base class for all other classes in pykickstart."""
- def __init__(self, *args, **kwargs):
- """Create a new KickstartObject instance. All other classes in
- pykickstart should be derived from this one. Instance attributes:
- """
- pass
-
- def __str__(self):
- return ""
diff --git a/scripts/lib/wic/3rdparty/pykickstart/options.py b/scripts/lib/wic/3rdparty/pykickstart/options.py
deleted file mode 100644
index b2d8e3e516..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/options.py
+++ /dev/null
@@ -1,223 +0,0 @@
-#
-# Chris Lumens <clumens@redhat.com>
-#
-# Copyright 2005, 2006, 2007 Red Hat, Inc.
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
-# trademarks that are incorporated in the source code or documentation are not
-# subject to the GNU General Public License and may only be used or replicated
-# with the express permission of Red Hat, Inc.
-#
-"""
-Specialized option handling.
-
-This module exports two classes:
-
- KSOptionParser - A specialized subclass of OptionParser to be used
- in BaseHandler subclasses.
-
- KSOption - A specialized subclass of Option.
-"""
-import warnings
-from copy import copy
-from optparse import *
-
-from constants import *
-from errors import *
-from version import *
-
-import gettext
-_ = lambda x: gettext.ldgettext("pykickstart", x)
-
-class KSOptionParser(OptionParser):
- """A specialized subclass of optparse.OptionParser to handle extra option
- attribute checking, work error reporting into the KickstartParseError
- framework, and to turn off the default help.
- """
- def exit(self, status=0, msg=None):
- pass
-
- def error(self, msg):
- if self.lineno != None:
- raise KickstartParseError, formatErrorMsg(self.lineno, msg=msg)
- else:
- raise KickstartParseError, msg
-
- def keys(self):
- retval = []
-
- for opt in self.option_list:
- if opt not in retval:
- retval.append(opt.dest)
-
- return retval
-
- def _init_parsing_state (self):
- OptionParser._init_parsing_state(self)
- self.option_seen = {}
-
- def check_values (self, values, args):
- def seen(self, option):
- return self.option_seen.has_key(option)
-
- def usedTooNew(self, option):
- return option.introduced and option.introduced > self.version
-
- def usedDeprecated(self, option):
- return option.deprecated
-
- def usedRemoved(self, option):
- return option.removed and option.removed <= self.version
-
- for option in filter(lambda o: isinstance(o, Option), self.option_list):
- if option.required and not seen(self, option):
- raise KickstartValueError, formatErrorMsg(self.lineno, _("Option %s is required") % option)
- elif seen(self, option) and usedTooNew(self, option):
- mapping = {"option": option, "intro": versionToString(option.introduced),
- "version": versionToString(self.version)}
- self.error(_("The %(option)s option was introduced in version %(intro)s, but you are using kickstart syntax version %(version)s.") % mapping)
- elif seen(self, option) and usedRemoved(self, option):
- mapping = {"option": option, "removed": versionToString(option.removed),
- "version": versionToString(self.version)}
-
- if option.removed == self.version:
- self.error(_("The %(option)s option is no longer supported.") % mapping)
- else:
- self.error(_("The %(option)s option was removed in version %(removed)s, but you are using kickstart syntax version %(version)s.") % mapping)
- elif seen(self, option) and usedDeprecated(self, option):
- mapping = {"lineno": self.lineno, "option": option}
- warnings.warn(_("Ignoring deprecated option on line %(lineno)s: The %(option)s option has been deprecated and no longer has any effect. It may be removed from future releases, which will result in a fatal error from kickstart. Please modify your kickstart file to remove this option.") % mapping, DeprecationWarning)
-
- return (values, args)
-
- def parse_args(self, *args, **kwargs):
- if kwargs.has_key("lineno"):
- self.lineno = kwargs.pop("lineno")
-
- return OptionParser.parse_args(self, **kwargs)
-
- def __init__(self, mapping=None, version=None):
- """Create a new KSOptionParser instance. Each KickstartCommand
- subclass should create one instance of KSOptionParser, providing
- at least the lineno attribute. mapping and version are not required.
- Instance attributes:
-
- mapping -- A mapping from option strings to different values.
- version -- The version of the kickstart syntax we are checking
- against.
- """
- OptionParser.__init__(self, option_class=KSOption,
- add_help_option=False,
- conflict_handler="resolve")
- if mapping is None:
- self.map = {}
- else:
- self.map = mapping
-
- self.lineno = None
- self.option_seen = {}
- self.version = version
-
-def _check_ksboolean(option, opt, value):
- if value.lower() in ("on", "yes", "true", "1"):
- return True
- elif value.lower() in ("off", "no", "false", "0"):
- return False
- else:
- mapping = {"opt": opt, "value": value}
- raise OptionValueError(_("Option %(opt)s: invalid boolean value: %(value)r") % mapping)
-
-def _check_string(option, opt, value):
- if len(value) > 2 and value.startswith("--"):
- mapping = {"opt": opt, "value": value}
- raise OptionValueError(_("Option %(opt)s: invalid string value: %(value)r") % mapping)
- else:
- return value
-
-def _check_size(option, opt, value):
- # Former default was MB
- if (value.isdigit()):
- return int(value) * 1024L
-
- mapping = {"opt": opt, "value": value}
- if (not value[:-1].isdigit()):
- raise OptionValueError(_("Option %(opt)s: invalid size value: %(value)r") % mapping)
-
- size = int(value[:-1])
- if (value.endswith("k") or value.endswith("K")):
- return size
- if (value.endswith("M")):
- return size * 1024L
- if (value.endswith("G")):
- return size * 1024L * 1024L
- raise OptionValueError(_("Option %(opt)s: invalid size value: %(value)r") % mapping)
-
-# Creates a new Option class that supports several new attributes:
-# - required: any option with this attribute must be supplied or an exception
-# is thrown
-# - introduced: the kickstart syntax version that this option first appeared
-# in - an exception will be raised if the option is used and
-# the specified syntax version is less than the value of this
-# attribute
-# - deprecated: the kickstart syntax version that this option was deprecated
-# in - a DeprecationWarning will be thrown if the option is
-# used and the specified syntax version is greater than the
-# value of this attribute
-# - removed: the kickstart syntax version that this option was removed in - an
-# exception will be raised if the option is used and the specified
-# syntax version is greated than the value of this attribute
-# Also creates a new type:
-# - ksboolean: support various kinds of boolean values on an option
-# And two new actions:
-# - map : allows you to define an opt -> val mapping such that dest gets val
-# when opt is seen
-# - map_extend: allows you to define an opt -> [val1, ... valn] mapping such
-# that dest gets a list of vals built up when opt is seen
-class KSOption (Option):
- ATTRS = Option.ATTRS + ['introduced', 'deprecated', 'removed', 'required']
- ACTIONS = Option.ACTIONS + ("map", "map_extend",)
- STORE_ACTIONS = Option.STORE_ACTIONS + ("map", "map_extend",)
-
- TYPES = Option.TYPES + ("ksboolean", "string", "size")
- TYPE_CHECKER = copy(Option.TYPE_CHECKER)
- TYPE_CHECKER["ksboolean"] = _check_ksboolean
- TYPE_CHECKER["string"] = _check_string
- TYPE_CHECKER["size"] = _check_size
-
- def _check_required(self):
- if self.required and not self.takes_value():
- raise OptionError(_("Required flag set for option that doesn't take a value"), self)
-
- # Make sure _check_required() is called from the constructor!
- CHECK_METHODS = Option.CHECK_METHODS + [_check_required]
-
- def process (self, opt, value, values, parser):
- Option.process(self, opt, value, values, parser)
- parser.option_seen[self] = 1
-
- # Override default take_action method to handle our custom actions.
- def take_action(self, action, dest, opt, value, values, parser):
- if action == "map":
- values.ensure_value(dest, parser.map[opt.lstrip('-')])
- elif action == "map_extend":
- values.ensure_value(dest, []).extend(parser.map[opt.lstrip('-')])
- else:
- Option.take_action(self, action, dest, opt, value, values, parser)
-
- def takes_value(self):
- # Deprecated options don't take a value.
- return Option.takes_value(self) and not self.deprecated
-
- def __init__(self, *args, **kwargs):
- self.deprecated = False
- self.required = False
- Option.__init__(self, *args, **kwargs)
diff --git a/scripts/lib/wic/3rdparty/pykickstart/parser.py b/scripts/lib/wic/3rdparty/pykickstart/parser.py
deleted file mode 100644
index 9c9674bf73..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/parser.py
+++ /dev/null
@@ -1,619 +0,0 @@
-#
-# parser.py: Kickstart file parser.
-#
-# Chris Lumens <clumens@redhat.com>
-#
-# Copyright 2005, 2006, 2007, 2008, 2011 Red Hat, Inc.
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
-# trademarks that are incorporated in the source code or documentation are not
-# subject to the GNU General Public License and may only be used or replicated
-# with the express permission of Red Hat, Inc.
-#
-"""
-Main kickstart file processing module.
-
-This module exports several important classes:
-
- Script - Representation of a single %pre, %post, or %traceback script.
-
- Packages - Representation of the %packages section.
-
- KickstartParser - The kickstart file parser state machine.
-"""
-
-from collections import Iterator
-import os
-import shlex
-import sys
-import tempfile
-from copy import copy
-from optparse import *
-
-import constants
-from errors import KickstartError, KickstartParseError, KickstartValueError, formatErrorMsg
-from ko import KickstartObject
-from sections import *
-import version
-
-import gettext
-_ = lambda x: gettext.ldgettext("pykickstart", x)
-
-STATE_END = "end"
-STATE_COMMANDS = "commands"
-
-ver = version.DEVEL
-
-
-class PutBackIterator(Iterator):
- def __init__(self, iterable):
- self._iterable = iter(iterable)
- self._buf = None
-
- def __iter__(self):
- return self
-
- def put(self, s):
- self._buf = s
-
- def next(self):
- if self._buf:
- retval = self._buf
- self._buf = None
- return retval
- else:
- return self._iterable.next()
-
-###
-### SCRIPT HANDLING
-###
-class Script(KickstartObject):
- """A class representing a single kickstart script. If functionality beyond
- just a data representation is needed (for example, a run method in
- anaconda), Script may be subclassed. Although a run method is not
- provided, most of the attributes of Script have to do with running the
- script. Instances of Script are held in a list by the Version object.
- """
- def __init__(self, script, *args , **kwargs):
- """Create a new Script instance. Instance attributes:
-
- errorOnFail -- If execution of the script fails, should anaconda
- stop, display an error, and then reboot without
- running any other scripts?
- inChroot -- Does the script execute in anaconda's chroot
- environment or not?
- interp -- The program that should be used to interpret this
- script.
- lineno -- The line number this script starts on.
- logfile -- Where all messages from the script should be logged.
- script -- A string containing all the lines of the script.
- type -- The type of the script, which can be KS_SCRIPT_* from
- pykickstart.constants.
- """
- KickstartObject.__init__(self, *args, **kwargs)
- self.script = "".join(script)
-
- self.interp = kwargs.get("interp", "/bin/sh")
- self.inChroot = kwargs.get("inChroot", False)
- self.lineno = kwargs.get("lineno", None)
- self.logfile = kwargs.get("logfile", None)
- self.errorOnFail = kwargs.get("errorOnFail", False)
- self.type = kwargs.get("type", constants.KS_SCRIPT_PRE)
-
- def __str__(self):
- """Return a string formatted for output to a kickstart file."""
- retval = ""
-
- if self.type == constants.KS_SCRIPT_PRE:
- retval += '\n%pre'
- elif self.type == constants.KS_SCRIPT_POST:
- retval += '\n%post'
- elif self.type == constants.KS_SCRIPT_TRACEBACK:
- retval += '\n%traceback'
-
- if self.interp != "/bin/sh" and self.interp != "":
- retval += " --interpreter=%s" % self.interp
- if self.type == constants.KS_SCRIPT_POST and not self.inChroot:
- retval += " --nochroot"
- if self.logfile != None:
- retval += " --logfile %s" % self.logfile
- if self.errorOnFail:
- retval += " --erroronfail"
-
- if self.script.endswith("\n"):
- if ver >= version.F8:
- return retval + "\n%s%%end\n" % self.script
- else:
- return retval + "\n%s\n" % self.script
- else:
- if ver >= version.F8:
- return retval + "\n%s\n%%end\n" % self.script
- else:
- return retval + "\n%s\n" % self.script
-
-
-##
-## PACKAGE HANDLING
-##
-class Group:
- """A class representing a single group in the %packages section."""
- def __init__(self, name="", include=constants.GROUP_DEFAULT):
- """Create a new Group instance. Instance attributes:
-
- name -- The group's identifier
- include -- The level of how much of the group should be included.
- Values can be GROUP_* from pykickstart.constants.
- """
- self.name = name
- self.include = include
-
- def __str__(self):
- """Return a string formatted for output to a kickstart file."""
- if self.include == constants.GROUP_REQUIRED:
- return "@%s --nodefaults" % self.name
- elif self.include == constants.GROUP_ALL:
- return "@%s --optional" % self.name
- else:
- return "@%s" % self.name
-
- def __cmp__(self, other):
- if self.name < other.name:
- return -1
- elif self.name > other.name:
- return 1
- return 0
-
-class Packages(KickstartObject):
- """A class representing the %packages section of the kickstart file."""
- def __init__(self, *args, **kwargs):
- """Create a new Packages instance. Instance attributes:
-
- addBase -- Should the Base group be installed even if it is
- not specified?
- default -- Should the default package set be selected?
- excludedList -- A list of all the packages marked for exclusion in
- the %packages section, without the leading minus
- symbol.
- excludeDocs -- Should documentation in each package be excluded?
- groupList -- A list of Group objects representing all the groups
- specified in the %packages section. Names will be
- stripped of the leading @ symbol.
- excludedGroupList -- A list of Group objects representing all the
- groups specified for removal in the %packages
- section. Names will be stripped of the leading
- -@ symbols.
- handleMissing -- If unknown packages are specified in the %packages
- section, should it be ignored or not? Values can
- be KS_MISSING_* from pykickstart.constants.
- packageList -- A list of all the packages specified in the
- %packages section.
- instLangs -- A list of languages to install.
- """
- KickstartObject.__init__(self, *args, **kwargs)
-
- self.addBase = True
- self.default = False
- self.excludedList = []
- self.excludedGroupList = []
- self.excludeDocs = False
- self.groupList = []
- self.handleMissing = constants.KS_MISSING_PROMPT
- self.packageList = []
- self.instLangs = None
-
- def __str__(self):
- """Return a string formatted for output to a kickstart file."""
- pkgs = ""
-
- if not self.default:
- grps = self.groupList
- grps.sort()
- for grp in grps:
- pkgs += "%s\n" % grp.__str__()
-
- p = self.packageList
- p.sort()
- for pkg in p:
- pkgs += "%s\n" % pkg
-
- grps = self.excludedGroupList
- grps.sort()
- for grp in grps:
- pkgs += "-%s\n" % grp.__str__()
-
- p = self.excludedList
- p.sort()
- for pkg in p:
- pkgs += "-%s\n" % pkg
-
- if pkgs == "":
- return ""
-
- retval = "\n%packages"
-
- if self.default:
- retval += " --default"
- if self.excludeDocs:
- retval += " --excludedocs"
- if not self.addBase:
- retval += " --nobase"
- if self.handleMissing == constants.KS_MISSING_IGNORE:
- retval += " --ignoremissing"
- if self.instLangs:
- retval += " --instLangs=%s" % self.instLangs
-
- if ver >= version.F8:
- return retval + "\n" + pkgs + "\n%end\n"
- else:
- return retval + "\n" + pkgs + "\n"
-
- def _processGroup (self, line):
- op = OptionParser()
- op.add_option("--nodefaults", action="store_true", default=False)
- op.add_option("--optional", action="store_true", default=False)
-
- (opts, extra) = op.parse_args(args=line.split())
-
- if opts.nodefaults and opts.optional:
- raise KickstartValueError, _("Group cannot specify both --nodefaults and --optional")
-
- # If the group name has spaces in it, we have to put it back together
- # now.
- grp = " ".join(extra)
-
- if opts.nodefaults:
- self.groupList.append(Group(name=grp, include=constants.GROUP_REQUIRED))
- elif opts.optional:
- self.groupList.append(Group(name=grp, include=constants.GROUP_ALL))
- else:
- self.groupList.append(Group(name=grp, include=constants.GROUP_DEFAULT))
-
- def add (self, pkgList):
- """Given a list of lines from the input file, strip off any leading
- symbols and add the result to the appropriate list.
- """
- existingExcludedSet = set(self.excludedList)
- existingPackageSet = set(self.packageList)
- newExcludedSet = set()
- newPackageSet = set()
-
- excludedGroupList = []
-
- for pkg in pkgList:
- stripped = pkg.strip()
-
- if stripped[0] == "@":
- self._processGroup(stripped[1:])
- elif stripped[0] == "-":
- if stripped[1] == "@":
- excludedGroupList.append(Group(name=stripped[2:]))
- else:
- newExcludedSet.add(stripped[1:])
- else:
- newPackageSet.add(stripped)
-
- # Groups have to be excluded in two different ways (note: can't use
- # sets here because we have to store objects):
- excludedGroupNames = map(lambda g: g.name, excludedGroupList)
-
- # First, an excluded group may be cancelling out a previously given
- # one. This is often the case when using %include. So there we should
- # just remove the group from the list.
- self.groupList = filter(lambda g: g.name not in excludedGroupNames, self.groupList)
-
- # Second, the package list could have included globs which are not
- # processed by pykickstart. In that case we need to preserve a list of
- # excluded groups so whatever tool doing package/group installation can
- # take appropriate action.
- self.excludedGroupList.extend(excludedGroupList)
-
- existingPackageSet = (existingPackageSet - newExcludedSet) | newPackageSet
- existingExcludedSet = (existingExcludedSet - existingPackageSet) | newExcludedSet
-
- self.packageList = list(existingPackageSet)
- self.excludedList = list(existingExcludedSet)
-
-
-###
-### PARSER
-###
-class KickstartParser:
- """The kickstart file parser class as represented by a basic state
- machine. To create a specialized parser, make a subclass and override
- any of the methods you care about. Methods that don't need to do
- anything may just pass. However, _stateMachine should never be
- overridden.
- """
- def __init__ (self, handler, followIncludes=True, errorsAreFatal=True,
- missingIncludeIsFatal=True):
- """Create a new KickstartParser instance. Instance attributes:
-
- errorsAreFatal -- Should errors cause processing to halt, or
- just print a message to the screen? This
- is most useful for writing syntax checkers
- that may want to continue after an error is
- encountered.
- followIncludes -- If %include is seen, should the included
- file be checked as well or skipped?
- handler -- An instance of a BaseHandler subclass. If
- None, the input file will still be parsed
- but no data will be saved and no commands
- will be executed.
- missingIncludeIsFatal -- Should missing include files be fatal, even
- if errorsAreFatal is False?
- """
- self.errorsAreFatal = errorsAreFatal
- self.followIncludes = followIncludes
- self.handler = handler
- self.currentdir = {}
- self.missingIncludeIsFatal = missingIncludeIsFatal
-
- self._state = STATE_COMMANDS
- self._includeDepth = 0
- self._line = ""
-
- self.version = self.handler.version
-
- global ver
- ver = self.version
-
- self._sections = {}
- self.setupSections()
-
- def _reset(self):
- """Reset the internal variables of the state machine for a new kickstart file."""
- self._state = STATE_COMMANDS
- self._includeDepth = 0
-
- def getSection(self, s):
- """Return a reference to the requested section (s must start with '%'s),
- or raise KeyError if not found.
- """
- return self._sections[s]
-
- def handleCommand (self, lineno, args):
- """Given the list of command and arguments, call the Version's
- dispatcher method to handle the command. Returns the command or
- data object returned by the dispatcher. This method may be
- overridden in a subclass if necessary.
- """
- if self.handler:
- self.handler.currentCmd = args[0]
- self.handler.currentLine = self._line
- retval = self.handler.dispatcher(args, lineno)
-
- return retval
-
- def registerSection(self, obj):
- """Given an instance of a Section subclass, register the new section
- with the parser. Calling this method means the parser will
- recognize your new section and dispatch into the given object to
- handle it.
- """
- if not obj.sectionOpen:
- raise TypeError, "no sectionOpen given for section %s" % obj
-
- if not obj.sectionOpen.startswith("%"):
- raise TypeError, "section %s tag does not start with a %%" % obj.sectionOpen
-
- self._sections[obj.sectionOpen] = obj
-
- def _finalize(self, obj):
- """Called at the close of a kickstart section to take any required
- actions. Internally, this is used to add scripts once we have the
- whole body read.
- """
- obj.finalize()
- self._state = STATE_COMMANDS
-
- def _handleSpecialComments(self, line):
- """Kickstart recognizes a couple special comments."""
- if self._state != STATE_COMMANDS:
- return
-
- # Save the platform for s-c-kickstart.
- if line[:10] == "#platform=":
- self.handler.platform = self._line[11:]
-
- def _readSection(self, lineIter, lineno):
- obj = self._sections[self._state]
-
- while True:
- try:
- line = lineIter.next()
- if line == "":
- # This section ends at the end of the file.
- if self.version >= version.F8:
- raise KickstartParseError, formatErrorMsg(lineno, msg=_("Section does not end with %%end."))
-
- self._finalize(obj)
- except StopIteration:
- break
-
- lineno += 1
-
- # Throw away blank lines and comments, unless the section wants all
- # lines.
- if self._isBlankOrComment(line) and not obj.allLines:
- continue
-
- if line.startswith("%"):
- args = shlex.split(line)
-
- if args and args[0] == "%end":
- # This is a properly terminated section.
- self._finalize(obj)
- break
- elif args and args[0] == "%ksappend":
- continue
- elif args and (self._validState(args[0]) or args[0] in ["%include", "%ksappend"]):
- # This is an unterminated section.
- if self.version >= version.F8:
- raise KickstartParseError, formatErrorMsg(lineno, msg=_("Section does not end with %%end."))
-
- # Finish up. We do not process the header here because
- # kicking back out to STATE_COMMANDS will ensure that happens.
- lineIter.put(line)
- lineno -= 1
- self._finalize(obj)
- break
- else:
- # This is just a line within a section. Pass it off to whatever
- # section handles it.
- obj.handleLine(line)
-
- return lineno
-
- def _validState(self, st):
- """Is the given section tag one that has been registered with the parser?"""
- return st in self._sections.keys()
-
- def _tryFunc(self, fn):
- """Call the provided function (which doesn't take any arguments) and
- do the appropriate error handling. If errorsAreFatal is False, this
- function will just print the exception and keep going.
- """
- try:
- fn()
- except Exception, msg:
- if self.errorsAreFatal:
- raise
- else:
- print msg
-
- def _isBlankOrComment(self, line):
- return line.isspace() or line == "" or line.lstrip()[0] == '#'
-
- def _stateMachine(self, lineIter):
- # For error reporting.
- lineno = 0
-
- while True:
- # Get the next line out of the file, quitting if this is the last line.
- try:
- self._line = lineIter.next()
- if self._line == "":
- break
- except StopIteration:
- break
-
- lineno += 1
-
- # Eliminate blank lines, whitespace-only lines, and comments.
- if self._isBlankOrComment(self._line):
- self._handleSpecialComments(self._line)
- continue
-
- # Remove any end-of-line comments.
- sanitized = self._line.split("#")[0]
-
- # Then split the line.
- args = shlex.split(sanitized.rstrip())
-
- if args[0] == "%include":
- # This case comes up primarily in ksvalidator.
- if not self.followIncludes:
- continue
-
- if len(args) == 1 or not args[1]:
- raise KickstartParseError, formatErrorMsg(lineno)
-
- self._includeDepth += 1
-
- try:
- self.readKickstart(args[1], reset=False)
- except KickstartError:
- # Handle the include file being provided over the
- # network in a %pre script. This case comes up in the
- # early parsing in anaconda.
- if self.missingIncludeIsFatal:
- raise
-
- self._includeDepth -= 1
- continue
-
- # Now on to the main event.
- if self._state == STATE_COMMANDS:
- if args[0] == "%ksappend":
- # This is handled by the preprocess* functions, so continue.
- continue
- elif args[0][0] == '%':
- # This is the beginning of a new section. Handle its header
- # here.
- newSection = args[0]
- if not self._validState(newSection):
- raise KickstartParseError, formatErrorMsg(lineno, msg=_("Unknown kickstart section: %s" % newSection))
-
- self._state = newSection
- obj = self._sections[self._state]
- self._tryFunc(lambda: obj.handleHeader(lineno, args))
-
- # This will handle all section processing, kicking us back
- # out to STATE_COMMANDS at the end with the current line
- # being the next section header, etc.
- lineno = self._readSection(lineIter, lineno)
- else:
- # This is a command in the command section. Dispatch to it.
- self._tryFunc(lambda: self.handleCommand(lineno, args))
- elif self._state == STATE_END:
- break
-
- def readKickstartFromString (self, s, reset=True):
- """Process a kickstart file, provided as the string str."""
- if reset:
- self._reset()
-
- # Add a "" to the end of the list so the string reader acts like the
- # file reader and we only get StopIteration when we're after the final
- # line of input.
- i = PutBackIterator(s.splitlines(True) + [""])
- self._stateMachine (i)
-
- def readKickstart(self, f, reset=True):
- """Process a kickstart file, given by the filename f."""
- if reset:
- self._reset()
-
- # an %include might not specify a full path. if we don't try to figure
- # out what the path should have been, then we're unable to find it
- # requiring full path specification, though, sucks. so let's make
- # the reading "smart" by keeping track of what the path is at each
- # include depth.
- if not os.path.exists(f):
- if self.currentdir.has_key(self._includeDepth - 1):
- if os.path.exists(os.path.join(self.currentdir[self._includeDepth - 1], f)):
- f = os.path.join(self.currentdir[self._includeDepth - 1], f)
-
- cd = os.path.dirname(f)
- if not cd.startswith("/"):
- cd = os.path.abspath(cd)
- self.currentdir[self._includeDepth] = cd
-
- try:
- s = file(f).read()
- except IOError, e:
- raise KickstartError, formatErrorMsg(0, msg=_("Unable to open input kickstart file: %s") % e.strerror)
-
- self.readKickstartFromString(s, reset=False)
-
- def setupSections(self):
- """Install the sections all kickstart files support. You may override
- this method in a subclass, but should avoid doing so unless you know
- what you're doing.
- """
- self._sections = {}
-
- # Install the sections all kickstart files support.
- self.registerSection(PreScriptSection(self.handler, dataObj=Script))
- self.registerSection(PostScriptSection(self.handler, dataObj=Script))
- self.registerSection(TracebackScriptSection(self.handler, dataObj=Script))
- self.registerSection(PackageSection(self.handler))
diff --git a/scripts/lib/wic/3rdparty/pykickstart/sections.py b/scripts/lib/wic/3rdparty/pykickstart/sections.py
deleted file mode 100644
index 44df856b8d..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/sections.py
+++ /dev/null
@@ -1,244 +0,0 @@
-#
-# sections.py: Kickstart file sections.
-#
-# Chris Lumens <clumens@redhat.com>
-#
-# Copyright 2011 Red Hat, Inc.
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
-# trademarks that are incorporated in the source code or documentation are not
-# subject to the GNU General Public License and may only be used or replicated
-# with the express permission of Red Hat, Inc.
-#
-"""
-This module exports the classes that define a section of a kickstart file. A
-section is a chunk of the file starting with a %tag and ending with a %end.
-Examples of sections include %packages, %pre, and %post.
-
-You may use this module to define your own custom sections which will be
-treated just the same as a predefined one by the kickstart parser. All that
-is necessary is to create a new subclass of Section and call
-parser.registerSection with an instance of your new class.
-"""
-from constants import *
-from options import KSOptionParser
-from version import *
-
-class Section(object):
- """The base class for defining kickstart sections. You are free to
- subclass this as appropriate.
-
- Class attributes:
-
- allLines -- Does this section require the parser to call handleLine
- for every line in the section, even blanks and comments?
- sectionOpen -- The string that denotes the start of this section. You
- must start your tag with a percent sign.
- timesSeen -- This attribute is for informational purposes only. It is
- incremented every time handleHeader is called to keep
- track of the number of times a section of this type is
- seen.
- """
- allLines = False
- sectionOpen = ""
- timesSeen = 0
-
- def __init__(self, handler, **kwargs):
- """Create a new Script instance. At the least, you must pass in an
- instance of a baseHandler subclass.
-
- Valid kwargs:
-
- dataObj --
- """
- self.handler = handler
-
- self.version = self.handler.version
-
- self.dataObj = kwargs.get("dataObj", None)
-
- def finalize(self):
- """This method is called when the %end tag for a section is seen. It
- is not required to be provided.
- """
- pass
-
- def handleLine(self, line):
- """This method is called for every line of a section. Take whatever
- action is appropriate. While this method is not required to be
- provided, not providing it does not make a whole lot of sense.
-
- Arguments:
-
- line -- The complete line, with any trailing newline.
- """
- pass
-
- def handleHeader(self, lineno, args):
- """This method is called when the opening tag for a section is seen.
- Not all sections will need this method, though all provided with
- kickstart include one.
-
- Arguments:
-
- args -- A list of all strings passed as arguments to the section
- opening tag.
- """
- self.timesSeen += 1
-
-class NullSection(Section):
- """This defines a section that pykickstart will recognize but do nothing
- with. If the parser runs across a %section that has no object registered,
- it will raise an error. Sometimes, you may want to simply ignore those
- sections instead. This class is useful for that purpose.
- """
- def __init__(self, *args, **kwargs):
- """Create a new NullSection instance. You must pass a sectionOpen
- parameter (including a leading '%') for the section you wish to
- ignore.
- """
- Section.__init__(self, *args, **kwargs)
- self.sectionOpen = kwargs.get("sectionOpen")
-
-class ScriptSection(Section):
- allLines = True
-
- def __init__(self, *args, **kwargs):
- Section.__init__(self, *args, **kwargs)
- self._script = {}
- self._resetScript()
-
- def _getParser(self):
- op = KSOptionParser(self.version)
- op.add_option("--erroronfail", dest="errorOnFail", action="store_true",
- default=False)
- op.add_option("--interpreter", dest="interpreter", default="/bin/sh")
- op.add_option("--log", "--logfile", dest="log")
- return op
-
- def _resetScript(self):
- self._script = {"interp": "/bin/sh", "log": None, "errorOnFail": False,
- "lineno": None, "chroot": False, "body": []}
-
- def handleLine(self, line):
- self._script["body"].append(line)
-
- def finalize(self):
- if " ".join(self._script["body"]).strip() == "":
- return
-
- kwargs = {"interp": self._script["interp"],
- "inChroot": self._script["chroot"],
- "lineno": self._script["lineno"],
- "logfile": self._script["log"],
- "errorOnFail": self._script["errorOnFail"],
- "type": self._script["type"]}
-
- s = self.dataObj (self._script["body"], **kwargs)
- self._resetScript()
-
- if self.handler:
- self.handler.scripts.append(s)
-
- def handleHeader(self, lineno, args):
- """Process the arguments to a %pre/%post/%traceback header for later
- setting on a Script instance once the end of the script is found.
- This method may be overridden in a subclass if necessary.
- """
- Section.handleHeader(self, lineno, args)
- op = self._getParser()
-
- (opts, extra) = op.parse_args(args=args[1:], lineno=lineno)
-
- self._script["interp"] = opts.interpreter
- self._script["lineno"] = lineno
- self._script["log"] = opts.log
- self._script["errorOnFail"] = opts.errorOnFail
- if hasattr(opts, "nochroot"):
- self._script["chroot"] = not opts.nochroot
-
-class PreScriptSection(ScriptSection):
- sectionOpen = "%pre"
-
- def _resetScript(self):
- ScriptSection._resetScript(self)
- self._script["type"] = KS_SCRIPT_PRE
-
-class PostScriptSection(ScriptSection):
- sectionOpen = "%post"
-
- def _getParser(self):
- op = ScriptSection._getParser(self)
- op.add_option("--nochroot", dest="nochroot", action="store_true",
- default=False)
- return op
-
- def _resetScript(self):
- ScriptSection._resetScript(self)
- self._script["chroot"] = True
- self._script["type"] = KS_SCRIPT_POST
-
-class TracebackScriptSection(ScriptSection):
- sectionOpen = "%traceback"
-
- def _resetScript(self):
- ScriptSection._resetScript(self)
- self._script["type"] = KS_SCRIPT_TRACEBACK
-
-class PackageSection(Section):
- sectionOpen = "%packages"
-
- def handleLine(self, line):
- if not self.handler:
- return
-
- (h, s, t) = line.partition('#')
- line = h.rstrip()
-
- self.handler.packages.add([line])
-
- def handleHeader(self, lineno, args):
- """Process the arguments to the %packages header and set attributes
- on the Version's Packages instance appropriate. This method may be
- overridden in a subclass if necessary.
- """
- Section.handleHeader(self, lineno, args)
- op = KSOptionParser(version=self.version)
- op.add_option("--excludedocs", dest="excludedocs", action="store_true",
- default=False)
- op.add_option("--ignoremissing", dest="ignoremissing",
- action="store_true", default=False)
- op.add_option("--nobase", dest="nobase", action="store_true",
- default=False)
- op.add_option("--ignoredeps", dest="resolveDeps", action="store_false",
- deprecated=FC4, removed=F9)
- op.add_option("--resolvedeps", dest="resolveDeps", action="store_true",
- deprecated=FC4, removed=F9)
- op.add_option("--default", dest="defaultPackages", action="store_true",
- default=False, introduced=F7)
- op.add_option("--instLangs", dest="instLangs", type="string",
- default="", introduced=F9)
-
- (opts, extra) = op.parse_args(args=args[1:], lineno=lineno)
-
- self.handler.packages.excludeDocs = opts.excludedocs
- self.handler.packages.addBase = not opts.nobase
- if opts.ignoremissing:
- self.handler.packages.handleMissing = KS_MISSING_IGNORE
- else:
- self.handler.packages.handleMissing = KS_MISSING_PROMPT
-
- if opts.defaultPackages:
- self.handler.packages.default = True
-
- if opts.instLangs:
- self.handler.packages.instLangs = opts.instLangs
diff --git a/scripts/lib/wic/3rdparty/pykickstart/version.py b/scripts/lib/wic/3rdparty/pykickstart/version.py
deleted file mode 100644
index 8a8e6aad22..0000000000
--- a/scripts/lib/wic/3rdparty/pykickstart/version.py
+++ /dev/null
@@ -1,168 +0,0 @@
-#
-# Chris Lumens <clumens@redhat.com>
-#
-# Copyright 2006, 2007, 2008, 2009, 2010 Red Hat, Inc.
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. Any Red Hat
-# trademarks that are incorporated in the source code or documentation are not
-# subject to the GNU General Public License and may only be used or replicated
-# with the express permission of Red Hat, Inc.
-#
-"""
-Methods for working with kickstart versions.
-
-This module defines several symbolic constants that specify kickstart syntax
-versions. Each version corresponds roughly to one release of Red Hat Linux,
-Red Hat Enterprise Linux, or Fedora Core as these are where most syntax
-changes take place.
-
-This module also exports several functions:
-
- makeVersion - Given a version number, return an instance of the
- matching handler class.
-
- returnClassForVersion - Given a version number, return the matching
- handler class. This does not return an
- instance of that class, however.
-
- stringToVersion - Convert a string representation of a version number
- into the symbolic constant.
-
- versionToString - Perform the reverse mapping.
-
- versionFromFile - Read a kickstart file and determine the version of
- syntax it uses. This requires the kickstart file to
- have a version= comment in it.
-"""
-import imputil, re, sys
-
-import gettext
-_ = lambda x: gettext.ldgettext("pykickstart", x)
-
-from pykickstart.errors import KickstartVersionError
-
-# Symbolic names for internal version numbers.
-RHEL3 = 900
-FC3 = 1000
-RHEL4 = 1100
-FC4 = 2000
-FC5 = 3000
-FC6 = 4000
-RHEL5 = 4100
-F7 = 5000
-F8 = 6000
-F9 = 7000
-F10 = 8000
-F11 = 9000
-F12 = 10000
-F13 = 11000
-RHEL6 = 11100
-F14 = 12000
-F15 = 13000
-F16 = 14000
-
-# This always points at the latest version and is the default.
-DEVEL = F16
-
-# A one-to-one mapping from string representations to version numbers.
-versionMap = {
- "DEVEL": DEVEL,
- "FC3": FC3, "FC4": FC4, "FC5": FC5, "FC6": FC6, "F7": F7, "F8": F8,
- "F9": F9, "F10": F10, "F11": F11, "F12": F12, "F13": F13,
- "F14": F14, "F15": F15, "F16": F16,
- "RHEL3": RHEL3, "RHEL4": RHEL4, "RHEL5": RHEL5, "RHEL6": RHEL6
-}
-
-def stringToVersion(s):
- """Convert string into one of the provided version constants. Raises
- KickstartVersionError if string does not match anything.
- """
- # First try these short forms.
- try:
- return versionMap[s.upper()]
- except KeyError:
- pass
-
- # Now try the Fedora versions.
- m = re.match("^fedora.* (\d+)$", s, re.I)
-
- if m and m.group(1):
- if versionMap.has_key("FC" + m.group(1)):
- return versionMap["FC" + m.group(1)]
- elif versionMap.has_key("F" + m.group(1)):
- return versionMap["F" + m.group(1)]
- else:
- raise KickstartVersionError(_("Unsupported version specified: %s") % s)
-
- # Now try the RHEL versions.
- m = re.match("^red hat enterprise linux.* (\d+)([\.\d]*)$", s, re.I)
-
- if m and m.group(1):
- if versionMap.has_key("RHEL" + m.group(1)):
- return versionMap["RHEL" + m.group(1)]
- else:
- raise KickstartVersionError(_("Unsupported version specified: %s") % s)
-
- # If nothing else worked, we're out of options.
- raise KickstartVersionError(_("Unsupported version specified: %s") % s)
-
-def versionToString(version, skipDevel=False):
- """Convert version into a string representation of the version number.
- This is the reverse operation of stringToVersion. Raises
- KickstartVersionError if version does not match anything.
- """
- if not skipDevel and version == versionMap["DEVEL"]:
- return "DEVEL"
-
- for (key, val) in versionMap.iteritems():
- if key == "DEVEL":
- continue
- elif val == version:
- return key
-
- raise KickstartVersionError(_("Unsupported version specified: %s") % version)
-
-def returnClassForVersion(version=DEVEL):
- """Return the class of the syntax handler for version. version can be
- either a string or the matching constant. Raises KickstartValueError
- if version does not match anything.
- """
- try:
- version = int(version)
- module = "%s" % versionToString(version, skipDevel=True)
- except ValueError:
- module = "%s" % version
- version = stringToVersion(version)
-
- module = module.lower()
-
- try:
- import pykickstart.handlers
- sys.path.extend(pykickstart.handlers.__path__)
- found = imputil.imp.find_module(module)
- loaded = imputil.imp.load_module(module, found[0], found[1], found[2])
-
- for (k, v) in loaded.__dict__.iteritems():
- if k.lower().endswith("%shandler" % module):
- return v
- except:
- raise KickstartVersionError(_("Unsupported version specified: %s") % version)
-
-def makeVersion(version=DEVEL):
- """Return a new instance of the syntax handler for version. version can be
- either a string or the matching constant. This function is useful for
- standalone programs which just need to handle a specific version of
- kickstart syntax (as provided by a command line argument, for example)
- and need to instantiate the correct object.
- """
- cl = returnClassForVersion(version)
- return cl()
diff --git a/scripts/lib/wic/__init__.py b/scripts/lib/wic/__init__.py
index 63c1d9c846..85876b138c 100644
--- a/scripts/lib/wic/__init__.py
+++ b/scripts/lib/wic/__init__.py
@@ -1,4 +1,20 @@
-import os, sys
+#!/usr/bin/env python -tt
+#
+# Copyright (c) 2007 Red Hat, Inc.
+# Copyright (c) 2011 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-cur_path = os.path.dirname(__file__) or '.'
-sys.path.insert(0, cur_path + '/3rdparty')
+class WicError(Exception):
+ pass
diff --git a/scripts/lib/wic/__version__.py b/scripts/lib/wic/__version__.py
deleted file mode 100644
index 5452a46712..0000000000
--- a/scripts/lib/wic/__version__.py
+++ /dev/null
@@ -1 +0,0 @@
-VERSION = "2.00"
diff --git a/scripts/lib/wic/canned-wks/common.wks.inc b/scripts/lib/wic/canned-wks/common.wks.inc
new file mode 100644
index 0000000000..5cf2fd1f3e
--- /dev/null
+++ b/scripts/lib/wic/canned-wks/common.wks.inc
@@ -0,0 +1,3 @@
+# This file is included into 3 canned wks files from this directory
+part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
+part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024
diff --git a/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg b/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg
new file mode 100644
index 0000000000..d5a07d2048
--- /dev/null
+++ b/scripts/lib/wic/canned-wks/directdisk-bootloader-config.cfg
@@ -0,0 +1,27 @@
+# This is an example configuration file for syslinux.
+TIMEOUT 50
+ALLOWOPTIONS 1
+SERIAL 0 115200
+PROMPT 0
+
+UI vesamenu.c32
+menu title Select boot options
+menu tabmsg Press [Tab] to edit, [Return] to select
+
+DEFAULT Graphics console boot
+
+LABEL Graphics console boot
+KERNEL /vmlinuz
+APPEND label=boot root=/dev/sda2 rootwait
+
+LABEL Serial console boot
+KERNEL /vmlinuz
+APPEND label=boot root=/dev/sda2 rootwait console=ttyS0,115200
+
+LABEL Graphics console install
+KERNEL /vmlinuz
+APPEND label=install root=/dev/sda2 rootwait
+
+LABEL Serial console install
+KERNEL /vmlinuz
+APPEND label=install root=/dev/sda2 rootwait console=ttyS0,115200
diff --git a/scripts/lib/wic/canned-wks/directdisk-bootloader-config.wks b/scripts/lib/wic/canned-wks/directdisk-bootloader-config.wks
new file mode 100644
index 0000000000..3529e05c87
--- /dev/null
+++ b/scripts/lib/wic/canned-wks/directdisk-bootloader-config.wks
@@ -0,0 +1,8 @@
+# short-description: Create a 'pcbios' direct disk image with custom bootloader config
+# long-description: Creates a partitioned legacy BIOS disk image that the user
+# can directly dd to boot media. The bootloader configuration source is a user file.
+
+include common.wks.inc
+
+bootloader --configfile="directdisk-bootloader-config.cfg"
+
diff --git a/scripts/lib/image/canned-wks/directdisk.wks b/scripts/lib/wic/canned-wks/directdisk-gpt.wks
index 62dcab15ce..8d7d8de6ea 100644
--- a/scripts/lib/image/canned-wks/directdisk.wks
+++ b/scripts/lib/wic/canned-wks/directdisk-gpt.wks
@@ -4,7 +4,7 @@
part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
-part / --source rootfs --ondisk sda --fstype=ext3 --label platform --align 1024
+part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
-bootloader --timeout=0 --append="rootwait rootfstype=ext3 video=vesafb vga=0x318 console=tty0"
+bootloader --ptable gpt --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8"
diff --git a/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks b/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks
new file mode 100644
index 0000000000..f61d941d6d
--- /dev/null
+++ b/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks
@@ -0,0 +1,23 @@
+# short-description: Create multi rootfs image using rootfs plugin
+# long-description: Creates a partitioned disk image with two rootfs partitions
+# using rootfs plugin.
+#
+# Partitions can use either
+# - indirect rootfs references to image recipe(s):
+# wic create directdisk-multi-indirect-recipes -e core-image-minimal \
+# --rootfs-dir rootfs1=core-image-minimal
+# --rootfs-dir rootfs2=core-image-minimal-dev
+#
+# - or paths to rootfs directories:
+# wic create directdisk-multi-rootfs \
+# --rootfs-dir rootfs1=tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs/
+# --rootfs-dir rootfs2=tmp/work/qemux86_64-poky-linux/core-image-minimal-dev/1.0-r0/rootfs/
+#
+# - or any combinations of -r and --rootfs command line options
+
+part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
+part / --source rootfs --rootfs-dir=rootfs1 --ondisk sda --fstype=ext4 --label platform --align 1024
+part /rescue --source rootfs --rootfs-dir=rootfs2 --ondisk sda --fstype=ext4 --label secondary --align 1024
+
+bootloader --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8"
+
diff --git a/scripts/lib/wic/canned-wks/directdisk.wks b/scripts/lib/wic/canned-wks/directdisk.wks
new file mode 100644
index 0000000000..8c8e06b02c
--- /dev/null
+++ b/scripts/lib/wic/canned-wks/directdisk.wks
@@ -0,0 +1,8 @@
+# short-description: Create a 'pcbios' direct disk image
+# long-description: Creates a partitioned legacy BIOS disk image that the user
+# can directly dd to boot media.
+
+include common.wks.inc
+
+bootloader --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8"
+
diff --git a/scripts/lib/image/canned-wks/mkefidisk.wks b/scripts/lib/wic/canned-wks/mkefidisk.wks
index 58d42e61eb..9f534fe184 100644
--- a/scripts/lib/image/canned-wks/mkefidisk.wks
+++ b/scripts/lib/wic/canned-wks/mkefidisk.wks
@@ -4,8 +4,8 @@
part /boot --source bootimg-efi --sourceparams="loader=grub-efi" --ondisk sda --label msdos --active --align 1024
-part / --source rootfs --ondisk sda --fstype=ext3 --label platform --align 1024
+part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
part swap --ondisk sda --size 44 --label swap1 --fstype=swap
-bootloader --timeout=10 --append="rootwait rootfstype=ext3 console=ttyPCH0,115200 console=tty0 vmalloc=256MB snd-hda-intel.enable_msi=0"
+bootloader --ptable gpt --timeout=5 --append="rootfstype=ext4 console=ttyS0,115200 console=tty0"
diff --git a/scripts/lib/wic/canned-wks/mkhybridiso.wks b/scripts/lib/wic/canned-wks/mkhybridiso.wks
new file mode 100644
index 0000000000..9d34e9b477
--- /dev/null
+++ b/scripts/lib/wic/canned-wks/mkhybridiso.wks
@@ -0,0 +1,7 @@
+# short-description: Create a hybrid ISO image
+# long-description: Creates an EFI and legacy bootable hybrid ISO image
+# which can be used on optical media as well as USB media.
+
+part /boot --source isoimage-isohybrid --sourceparams="loader=grub-efi,image_name=HYBRID_ISO_IMG" --ondisk cd --label HYBRIDISO --fstype=ext4
+
+bootloader --timeout=15 --append=""
diff --git a/scripts/lib/wic/canned-wks/qemux86-directdisk.wks b/scripts/lib/wic/canned-wks/qemux86-directdisk.wks
new file mode 100644
index 0000000000..db30bbced0
--- /dev/null
+++ b/scripts/lib/wic/canned-wks/qemux86-directdisk.wks
@@ -0,0 +1,8 @@
+# short-description: Create a qemu machine 'pcbios' direct disk image
+# long-description: Creates a partitioned legacy BIOS disk image that the user
+# can directly use to boot a qemu machine.
+
+include common.wks.inc
+
+bootloader --timeout=0 --append="vga=0 uvesafb.mode_option=640x480-32 root=/dev/sda2 rw mem=256M ip=192.168.7.2::192.168.7.1:255.255.255.0 oprofile.timer=1 rootfstype=ext4 "
+
diff --git a/scripts/lib/image/canned-wks/sdimage-bootpart.wks b/scripts/lib/wic/canned-wks/sdimage-bootpart.wks
index 7ffd632f4a..7ffd632f4a 100644
--- a/scripts/lib/image/canned-wks/sdimage-bootpart.wks
+++ b/scripts/lib/wic/canned-wks/sdimage-bootpart.wks
diff --git a/scripts/lib/wic/canned-wks/systemd-bootdisk.wks b/scripts/lib/wic/canned-wks/systemd-bootdisk.wks
new file mode 100644
index 0000000000..4bd9d6a65f
--- /dev/null
+++ b/scripts/lib/wic/canned-wks/systemd-bootdisk.wks
@@ -0,0 +1,11 @@
+# short-description: Create an EFI disk image with systemd-boot
+# long-description: Creates a partitioned EFI disk image that the user
+# can directly dd to boot media. The selected bootloader is systemd-boot.
+
+part /boot --source bootimg-efi --sourceparams="loader=systemd-boot" --ondisk sda --label msdos --active --align 1024
+
+part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
+
+part swap --ondisk sda --size 44 --label swap1 --fstype=swap
+
+bootloader --ptable gpt --timeout=5 --append="rootwait rootfstype=ext4 console=ttyS0,115200 console=tty0"
diff --git a/scripts/lib/wic/conf.py b/scripts/lib/wic/conf.py
deleted file mode 100644
index be34355ce4..0000000000
--- a/scripts/lib/wic/conf.py
+++ /dev/null
@@ -1,102 +0,0 @@
-#!/usr/bin/env python -tt
-#
-# Copyright (c) 2011 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-import os, sys, re
-import ConfigParser
-
-from wic import msger
-from wic import kickstart
-from wic.utils import misc, runner, errors
-
-
-def get_siteconf():
- wic_path = os.path.dirname(__file__)
- eos = wic_path.find('scripts') + len('scripts')
- scripts_path = wic_path[:eos]
-
- return scripts_path + "/lib/image/config/wic.conf"
-
-class ConfigMgr(object):
- DEFAULTS = {'common': {
- "distro_name": "Default Distribution",
- "plugin_dir": "/usr/lib/wic/plugins", # TODO use prefix also?
- },
- 'create': {
- "tmpdir": '/var/tmp/wic',
- "outdir": './wic-output',
-
- "release": None,
- "logfile": None,
- "name_prefix": None,
- "name_suffix": None,
- },
- }
-
- # make the manager class as singleton
- _instance = None
- def __new__(cls, *args, **kwargs):
- if not cls._instance:
- cls._instance = super(ConfigMgr, cls).__new__(cls, *args, **kwargs)
-
- return cls._instance
-
- def __init__(self, ksconf=None, siteconf=None):
- # reset config options
- self.reset()
-
- if not siteconf:
- siteconf = get_siteconf()
-
- # initial options from siteconf
- self._siteconf = siteconf
-
- if ksconf:
- self._ksconf = ksconf
-
- def reset(self):
- self.__ksconf = None
- self.__siteconf = None
-
- # initialize the values with defaults
- for sec, vals in self.DEFAULTS.iteritems():
- setattr(self, sec, vals)
-
- def __set_ksconf(self, ksconf):
- if not os.path.isfile(ksconf):
- msger.error('Cannot find ks file: %s' % ksconf)
-
- self.__ksconf = ksconf
- self._parse_kickstart(ksconf)
- def __get_ksconf(self):
- return self.__ksconf
- _ksconf = property(__get_ksconf, __set_ksconf)
-
- def _parse_kickstart(self, ksconf=None):
- if not ksconf:
- return
-
- ks = kickstart.read_kickstart(ksconf)
-
- self.create['ks'] = ks
- self.create['name'] = os.path.splitext(os.path.basename(ksconf))[0]
-
- self.create['name'] = misc.build_name(ksconf,
- self.create['release'],
- self.create['name_prefix'],
- self.create['name_suffix'])
-
-configmgr = ConfigMgr()
diff --git a/scripts/lib/wic/creator.py b/scripts/lib/wic/creator.py
deleted file mode 100644
index 2219377b38..0000000000
--- a/scripts/lib/wic/creator.py
+++ /dev/null
@@ -1,187 +0,0 @@
-#!/usr/bin/env python -tt
-#
-# Copyright (c) 2011 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-import os, sys, re
-from optparse import SUPPRESS_HELP
-
-from wic import msger
-from wic.utils import cmdln, errors
-from wic.conf import configmgr
-from wic.plugin import pluginmgr
-
-
-class Creator(cmdln.Cmdln):
- """${name}: create an image
-
- Usage:
- ${name} SUBCOMMAND <ksfile> [OPTS]
-
- ${command_list}
- ${option_list}
- """
-
- name = 'wic create(cr)'
-
- def __init__(self, *args, **kwargs):
- cmdln.Cmdln.__init__(self, *args, **kwargs)
- self._subcmds = []
-
- # get cmds from pluginmgr
- # mix-in do_subcmd interface
- for subcmd, klass in pluginmgr.get_plugins('imager').iteritems():
- if not hasattr(klass, 'do_create'):
- msger.warning("Unsupported subcmd: %s" % subcmd)
- continue
-
- func = getattr(klass, 'do_create')
- setattr(self.__class__, "do_"+subcmd, func)
- self._subcmds.append(subcmd)
-
- def get_optparser(self):
- optparser = cmdln.CmdlnOptionParser(self)
- optparser.add_option('-d', '--debug', action='store_true',
- dest='debug',
- help=SUPPRESS_HELP)
- optparser.add_option('-v', '--verbose', action='store_true',
- dest='verbose',
- help=SUPPRESS_HELP)
- optparser.add_option('', '--logfile', type='string', dest='logfile',
- default=None,
- help='Path of logfile')
- optparser.add_option('-c', '--config', type='string', dest='config',
- default=None,
- help='Specify config file for wic')
- optparser.add_option('-o', '--outdir', type='string', action='store',
- dest='outdir', default=None,
- help='Output directory')
- optparser.add_option('', '--tmpfs', action='store_true', dest='enabletmpfs',
- help='Setup tmpdir as tmpfs to accelerate, experimental'
- ' feature, use it if you have more than 4G memory')
- return optparser
-
- def preoptparse(self, argv):
- optparser = self.get_optparser()
-
- largs = []
- rargs = []
- while argv:
- arg = argv.pop(0)
-
- if arg in ('-h', '--help'):
- rargs.append(arg)
-
- elif optparser.has_option(arg):
- largs.append(arg)
-
- if optparser.get_option(arg).takes_value():
- try:
- largs.append(argv.pop(0))
- except IndexError:
- raise errors.Usage("option %s requires arguments" % arg)
-
- else:
- if arg.startswith("--"):
- if "=" in arg:
- opt = arg.split("=")[0]
- else:
- opt = None
- elif arg.startswith("-") and len(arg) > 2:
- opt = arg[0:2]
- else:
- opt = None
-
- if opt and optparser.has_option(opt):
- largs.append(arg)
- else:
- rargs.append(arg)
-
- return largs + rargs
-
- def postoptparse(self):
- abspath = lambda pth: os.path.abspath(os.path.expanduser(pth))
-
- if self.options.verbose:
- msger.set_loglevel('verbose')
- if self.options.debug:
- msger.set_loglevel('debug')
-
- if self.options.logfile:
- logfile_abs_path = abspath(self.options.logfile)
- if os.path.isdir(logfile_abs_path):
- raise errors.Usage("logfile's path %s should be file"
- % self.options.logfile)
- if not os.path.exists(os.path.dirname(logfile_abs_path)):
- os.makedirs(os.path.dirname(logfile_abs_path))
- msger.set_interactive(False)
- msger.set_logfile(logfile_abs_path)
- configmgr.create['logfile'] = self.options.logfile
-
- if self.options.config:
- configmgr.reset()
- configmgr._siteconf = self.options.config
-
- if self.options.outdir is not None:
- configmgr.create['outdir'] = abspath(self.options.outdir)
-
- cdir = 'outdir'
- if os.path.exists(configmgr.create[cdir]) \
- and not os.path.isdir(configmgr.create[cdir]):
- msger.error('Invalid directory specified: %s' \
- % configmgr.create[cdir])
-
- if self.options.enabletmpfs:
- configmgr.create['enabletmpfs'] = self.options.enabletmpfs
-
- def main(self, argv=None):
- if argv is None:
- argv = sys.argv
- else:
- argv = argv[:] # don't modify caller's list
-
- self.optparser = self.get_optparser()
- if self.optparser:
- try:
- argv = self.preoptparse(argv)
- self.options, args = self.optparser.parse_args(argv)
-
- except cmdln.CmdlnUserError, ex:
- msg = "%s: %s\nTry '%s help' for info.\n"\
- % (self.name, ex, self.name)
- msger.error(msg)
-
- except cmdln.StopOptionProcessing, ex:
- return 0
- else:
- # optparser=None means no process for opts
- self.options, args = None, argv[1:]
-
- if not args:
- return self.emptyline()
-
- self.postoptparse()
-
- return self.cmd(args)
-
- def precmd(self, argv): # check help before cmd
-
- if '-h' in argv or '?' in argv or '--help' in argv or 'help' in argv:
- return argv
-
- if len(argv) == 1:
- return ['help', argv[0]]
-
- return argv
diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
new file mode 100644
index 0000000000..f59821fea6
--- /dev/null
+++ b/scripts/lib/wic/engine.py
@@ -0,0 +1,257 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (c) 2013, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+
+# This module implements the image creation engine used by 'wic' to
+# create images. The engine parses through the OpenEmbedded kickstart
+# (wks) file specified and generates images that can then be directly
+# written onto media.
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+#
+
+import logging
+import os
+
+from wic import WicError
+from wic.pluginbase import PluginMgr
+from wic.utils.misc import get_bitbake_var
+
+logger = logging.getLogger('wic')
+
+def verify_build_env():
+ """
+ Verify that the build environment is sane.
+
+ Returns True if it is, false otherwise
+ """
+ if not os.environ.get("BUILDDIR"):
+ raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)")
+
+ return True
+
+
+CANNED_IMAGE_DIR = "lib/wic/canned-wks" # relative to scripts
+SCRIPTS_CANNED_IMAGE_DIR = "scripts/" + CANNED_IMAGE_DIR
+WIC_DIR = "wic"
+
+def build_canned_image_list(path):
+ layers_path = get_bitbake_var("BBLAYERS")
+ canned_wks_layer_dirs = []
+
+ if layers_path is not None:
+ for layer_path in layers_path.split():
+ for wks_path in (WIC_DIR, SCRIPTS_CANNED_IMAGE_DIR):
+ cpath = os.path.join(layer_path, wks_path)
+ if os.path.isdir(cpath):
+ canned_wks_layer_dirs.append(cpath)
+
+ cpath = os.path.join(path, CANNED_IMAGE_DIR)
+ canned_wks_layer_dirs.append(cpath)
+
+ return canned_wks_layer_dirs
+
+def find_canned_image(scripts_path, wks_file):
+ """
+ Find a .wks file with the given name in the canned files dir.
+
+ Return False if not found
+ """
+ layers_canned_wks_dir = build_canned_image_list(scripts_path)
+
+ for canned_wks_dir in layers_canned_wks_dir:
+ for root, dirs, files in os.walk(canned_wks_dir):
+ for fname in files:
+ if fname.endswith("~") or fname.endswith("#"):
+ continue
+ if fname.endswith(".wks") and wks_file + ".wks" == fname:
+ fullpath = os.path.join(canned_wks_dir, fname)
+ return fullpath
+ return None
+
+
+def list_canned_images(scripts_path):
+ """
+ List the .wks files in the canned image dir, minus the extension.
+ """
+ layers_canned_wks_dir = build_canned_image_list(scripts_path)
+
+ for canned_wks_dir in layers_canned_wks_dir:
+ for root, dirs, files in os.walk(canned_wks_dir):
+ for fname in files:
+ if fname.endswith("~") or fname.endswith("#"):
+ continue
+ if fname.endswith(".wks"):
+ fullpath = os.path.join(canned_wks_dir, fname)
+ with open(fullpath) as wks:
+ for line in wks:
+ desc = ""
+ idx = line.find("short-description:")
+ if idx != -1:
+ desc = line[idx + len("short-description:"):].strip()
+ break
+ basename = os.path.splitext(fname)[0]
+ print(" %s\t\t%s" % (basename.ljust(30), desc))
+
+
+def list_canned_image_help(scripts_path, fullpath):
+ """
+ List the help and params in the specified canned image.
+ """
+ found = False
+ with open(fullpath) as wks:
+ for line in wks:
+ if not found:
+ idx = line.find("long-description:")
+ if idx != -1:
+ print()
+ print(line[idx + len("long-description:"):].strip())
+ found = True
+ continue
+ if not line.strip():
+ break
+ idx = line.find("#")
+ if idx != -1:
+ print(line[idx + len("#:"):].rstrip())
+ else:
+ break
+
+
+def list_source_plugins():
+ """
+ List the available source plugins i.e. plugins available for --source.
+ """
+ plugins = PluginMgr.get_plugins('source')
+
+ for plugin in plugins:
+ print(" %s" % plugin)
+
+
+def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir,
+ native_sysroot, options):
+ """
+ Create image
+
+ wks_file - user-defined OE kickstart file
+ rootfs_dir - absolute path to the build's /rootfs dir
+ bootimg_dir - absolute path to the build's boot artifacts directory
+ kernel_dir - absolute path to the build's kernel directory
+ native_sysroot - absolute path to the build's native sysroots dir
+ image_output_dir - dirname to create for image
+ options - wic command line options (debug, bmap, etc)
+
+ Normally, the values for the build artifacts values are determined
+ by 'wic -e' from the output of the 'bitbake -e' command given an
+ image name e.g. 'core-image-minimal' and a given machine set in
+ local.conf. If that's the case, the variables get the following
+ values from the output of 'bitbake -e':
+
+ rootfs_dir: IMAGE_ROOTFS
+ kernel_dir: DEPLOY_DIR_IMAGE
+ native_sysroot: STAGING_DIR_NATIVE
+
+ In the above case, bootimg_dir remains unset and the
+ plugin-specific image creation code is responsible for finding the
+ bootimg artifacts.
+
+ In the case where the values are passed in explicitly i.e 'wic -e'
+ is not used but rather the individual 'wic' options are used to
+ explicitly specify these values.
+ """
+ try:
+ oe_builddir = os.environ["BUILDDIR"]
+ except KeyError:
+ raise WicError("BUILDDIR not found, exiting. (Did you forget to source oe-init-build-env?)")
+
+ if not os.path.exists(options.outdir):
+ os.makedirs(options.outdir)
+
+ pname = 'direct'
+ plugin_class = PluginMgr.get_plugins('imager').get(pname)
+ if not plugin_class:
+ raise WicError('Unknown plugin: %s' % pname)
+
+ plugin = plugin_class(wks_file, rootfs_dir, bootimg_dir, kernel_dir,
+ native_sysroot, oe_builddir, options)
+
+ plugin.do_create()
+
+ logger.info("The image(s) were created using OE kickstart file:\n %s", wks_file)
+
+
+def wic_list(args, scripts_path):
+ """
+ Print the list of images or source plugins.
+ """
+ if len(args) < 1:
+ return False
+
+ if args == ["images"]:
+ list_canned_images(scripts_path)
+ return True
+ elif args == ["source-plugins"]:
+ list_source_plugins()
+ return True
+ elif len(args) == 2 and args[1] == "help":
+ wks_file = args[0]
+ fullpath = find_canned_image(scripts_path, wks_file)
+ if not fullpath:
+ raise WicError("No image named %s found, exiting. "
+ "(Use 'wic list images' to list available images, "
+ "or specify a fully-qualified OE kickstart (.wks) "
+ "filename)" % wks_file)
+
+ list_canned_image_help(scripts_path, fullpath)
+ return True
+
+ return False
+
+def find_canned(scripts_path, file_name):
+ """
+ Find a file either by its path or by name in the canned files dir.
+
+ Return None if not found
+ """
+ if os.path.exists(file_name):
+ return file_name
+
+ layers_canned_wks_dir = build_canned_image_list(scripts_path)
+ for canned_wks_dir in layers_canned_wks_dir:
+ for root, dirs, files in os.walk(canned_wks_dir):
+ for fname in files:
+ if fname == file_name:
+ fullpath = os.path.join(canned_wks_dir, fname)
+ return fullpath
+
+def get_custom_config(boot_file):
+ """
+ Get the custom configuration to be used for the bootloader.
+
+ Return None if the file can't be found.
+ """
+ # Get the scripts path of poky
+ scripts_path = os.path.abspath("%s/../.." % os.path.dirname(__file__))
+
+ cfg_file = find_canned(scripts_path, boot_file)
+ if cfg_file:
+ with open(cfg_file, "r") as f:
+ config = f.read()
+ return config
diff --git a/scripts/lib/wic/filemap.py b/scripts/lib/wic/filemap.py
new file mode 100644
index 0000000000..1f1aacc522
--- /dev/null
+++ b/scripts/lib/wic/filemap.py
@@ -0,0 +1,564 @@
+# Copyright (c) 2012 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License, version 2,
+# as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+
+"""
+This module implements python implements a way to get file block. Two methods
+are supported - the FIEMAP ioctl and the 'SEEK_HOLE / SEEK_DATA' features of
+the file seek syscall. The former is implemented by the 'FilemapFiemap' class,
+the latter is implemented by the 'FilemapSeek' class. Both classes provide the
+same API. The 'filemap' function automatically selects which class can be used
+and returns an instance of the class.
+"""
+
+# Disable the following pylint recommendations:
+# * Too many instance attributes (R0902)
+# pylint: disable=R0902
+
+import os
+import struct
+import array
+import fcntl
+import tempfile
+import logging
+
+def get_block_size(file_obj):
+ """
+ Returns block size for file object 'file_obj'. Errors are indicated by the
+ 'IOError' exception.
+ """
+
+ from fcntl import ioctl
+ import struct
+
+ # Get the block size of the host file-system for the image file by calling
+ # the FIGETBSZ ioctl (number 2).
+ binary_data = ioctl(file_obj, 2, struct.pack('I', 0))
+ return struct.unpack('I', binary_data)[0]
+
+class ErrorNotSupp(Exception):
+ """
+ An exception of this type is raised when the 'FIEMAP' or 'SEEK_HOLE' feature
+ is not supported either by the kernel or the file-system.
+ """
+ pass
+
+class Error(Exception):
+ """A class for all the other exceptions raised by this module."""
+ pass
+
+
+class _FilemapBase(object):
+ """
+ This is a base class for a couple of other classes in this module. This
+ class simply performs the common parts of the initialization process: opens
+ the image file, gets its size, etc. The 'log' parameter is the logger object
+ to use for printing messages.
+ """
+
+ def __init__(self, image, log=None):
+ """
+ Initialize a class instance. The 'image' argument is full path to the
+ file or file object to operate on.
+ """
+
+ self._log = log
+ if self._log is None:
+ self._log = logging.getLogger(__name__)
+
+ self._f_image_needs_close = False
+
+ if hasattr(image, "fileno"):
+ self._f_image = image
+ self._image_path = image.name
+ else:
+ self._image_path = image
+ self._open_image_file()
+
+ try:
+ self.image_size = os.fstat(self._f_image.fileno()).st_size
+ except IOError as err:
+ raise Error("cannot get information about file '%s': %s"
+ % (self._f_image.name, err))
+
+ try:
+ self.block_size = get_block_size(self._f_image)
+ except IOError as err:
+ raise Error("cannot get block size for '%s': %s"
+ % (self._image_path, err))
+
+ self.blocks_cnt = self.image_size + self.block_size - 1
+ self.blocks_cnt //= self.block_size
+
+ try:
+ self._f_image.flush()
+ except IOError as err:
+ raise Error("cannot flush image file '%s': %s"
+ % (self._image_path, err))
+
+ try:
+ os.fsync(self._f_image.fileno()),
+ except OSError as err:
+ raise Error("cannot synchronize image file '%s': %s "
+ % (self._image_path, err.strerror))
+
+ self._log.debug("opened image \"%s\"" % self._image_path)
+ self._log.debug("block size %d, blocks count %d, image size %d"
+ % (self.block_size, self.blocks_cnt, self.image_size))
+
+ def __del__(self):
+ """The class destructor which just closes the image file."""
+ if self._f_image_needs_close:
+ self._f_image.close()
+
+ def _open_image_file(self):
+ """Open the image file."""
+ try:
+ self._f_image = open(self._image_path, 'rb')
+ except IOError as err:
+ raise Error("cannot open image file '%s': %s"
+ % (self._image_path, err))
+
+ self._f_image_needs_close = True
+
+ def block_is_mapped(self, block): # pylint: disable=W0613,R0201
+ """
+ This method has has to be implemented by child classes. It returns
+ 'True' if block number 'block' of the image file is mapped and 'False'
+ otherwise.
+ """
+
+ raise Error("the method is not implemented")
+
+ def block_is_unmapped(self, block): # pylint: disable=W0613,R0201
+ """
+ This method has has to be implemented by child classes. It returns
+ 'True' if block number 'block' of the image file is not mapped (hole)
+ and 'False' otherwise.
+ """
+
+ raise Error("the method is not implemented")
+
+ def get_mapped_ranges(self, start, count): # pylint: disable=W0613,R0201
+ """
+ This method has has to be implemented by child classes. This is a
+ generator which yields ranges of mapped blocks in the file. The ranges
+ are tuples of 2 elements: [first, last], where 'first' is the first
+ mapped block and 'last' is the last mapped block.
+
+ The ranges are yielded for the area of the file of size 'count' blocks,
+ starting from block 'start'.
+ """
+
+ raise Error("the method is not implemented")
+
+ def get_unmapped_ranges(self, start, count): # pylint: disable=W0613,R0201
+ """
+ This method has has to be implemented by child classes. Just like
+ 'get_mapped_ranges()', but yields unmapped block ranges instead
+ (holes).
+ """
+
+ raise Error("the method is not implemented")
+
+
+# The 'SEEK_HOLE' and 'SEEK_DATA' options of the file seek system call
+_SEEK_DATA = 3
+_SEEK_HOLE = 4
+
+def _lseek(file_obj, offset, whence):
+ """This is a helper function which invokes 'os.lseek' for file object
+ 'file_obj' and with specified 'offset' and 'whence'. The 'whence'
+ argument is supposed to be either '_SEEK_DATA' or '_SEEK_HOLE'. When
+ there is no more data or hole starting from 'offset', this function
+ returns '-1'. Otherwise the data or hole position is returned."""
+
+ try:
+ return os.lseek(file_obj.fileno(), offset, whence)
+ except OSError as err:
+ # The 'lseek' system call returns the ENXIO if there is no data or
+ # hole starting from the specified offset.
+ if err.errno == os.errno.ENXIO:
+ return -1
+ elif err.errno == os.errno.EINVAL:
+ raise ErrorNotSupp("the kernel or file-system does not support "
+ "\"SEEK_HOLE\" and \"SEEK_DATA\"")
+ else:
+ raise
+
+class FilemapSeek(_FilemapBase):
+ """
+ This class uses the 'SEEK_HOLE' and 'SEEK_DATA' to find file block mapping.
+ Unfortunately, the current implementation requires the caller to have write
+ access to the image file.
+ """
+
+ def __init__(self, image, log=None):
+ """Refer the '_FilemapBase' class for the documentation."""
+
+ # Call the base class constructor first
+ _FilemapBase.__init__(self, image, log)
+ self._log.debug("FilemapSeek: initializing")
+
+ self._probe_seek_hole()
+
+ def _probe_seek_hole(self):
+ """
+ Check whether the system implements 'SEEK_HOLE' and 'SEEK_DATA'.
+ Unfortunately, there seems to be no clean way for detecting this,
+ because often the system just fakes them by just assuming that all
+ files are fully mapped, so 'SEEK_HOLE' always returns EOF and
+ 'SEEK_DATA' always returns the requested offset.
+
+ I could not invent a better way of detecting the fake 'SEEK_HOLE'
+ implementation than just to create a temporary file in the same
+ directory where the image file resides. It would be nice to change this
+ to something better.
+ """
+
+ directory = os.path.dirname(self._image_path)
+
+ try:
+ tmp_obj = tempfile.TemporaryFile("w+", dir=directory)
+ except IOError as err:
+ raise ErrorNotSupp("cannot create a temporary in \"%s\": %s"
+ % (directory, err))
+
+ try:
+ os.ftruncate(tmp_obj.fileno(), self.block_size)
+ except OSError as err:
+ raise ErrorNotSupp("cannot truncate temporary file in \"%s\": %s"
+ % (directory, err))
+
+ offs = _lseek(tmp_obj, 0, _SEEK_HOLE)
+ if offs != 0:
+ # We are dealing with the stub 'SEEK_HOLE' implementation which
+ # always returns EOF.
+ self._log.debug("lseek(0, SEEK_HOLE) returned %d" % offs)
+ raise ErrorNotSupp("the file-system does not support "
+ "\"SEEK_HOLE\" and \"SEEK_DATA\" but only "
+ "provides a stub implementation")
+
+ tmp_obj.close()
+
+ def block_is_mapped(self, block):
+ """Refer the '_FilemapBase' class for the documentation."""
+ offs = _lseek(self._f_image, block * self.block_size, _SEEK_DATA)
+ if offs == -1:
+ result = False
+ else:
+ result = (offs // self.block_size == block)
+
+ self._log.debug("FilemapSeek: block_is_mapped(%d) returns %s"
+ % (block, result))
+ return result
+
+ def block_is_unmapped(self, block):
+ """Refer the '_FilemapBase' class for the documentation."""
+ return not self.block_is_mapped(block)
+
+ def _get_ranges(self, start, count, whence1, whence2):
+ """
+ This function implements 'get_mapped_ranges()' and
+ 'get_unmapped_ranges()' depending on what is passed in the 'whence1'
+ and 'whence2' arguments.
+ """
+
+ assert whence1 != whence2
+ end = start * self.block_size
+ limit = end + count * self.block_size
+
+ while True:
+ start = _lseek(self._f_image, end, whence1)
+ if start == -1 or start >= limit or start == self.image_size:
+ break
+
+ end = _lseek(self._f_image, start, whence2)
+ if end == -1 or end == self.image_size:
+ end = self.blocks_cnt * self.block_size
+ if end > limit:
+ end = limit
+
+ start_blk = start // self.block_size
+ end_blk = end // self.block_size - 1
+ self._log.debug("FilemapSeek: yielding range (%d, %d)"
+ % (start_blk, end_blk))
+ yield (start_blk, end_blk)
+
+ def get_mapped_ranges(self, start, count):
+ """Refer the '_FilemapBase' class for the documentation."""
+ self._log.debug("FilemapSeek: get_mapped_ranges(%d, %d(%d))"
+ % (start, count, start + count - 1))
+ return self._get_ranges(start, count, _SEEK_DATA, _SEEK_HOLE)
+
+ def get_unmapped_ranges(self, start, count):
+ """Refer the '_FilemapBase' class for the documentation."""
+ self._log.debug("FilemapSeek: get_unmapped_ranges(%d, %d(%d))"
+ % (start, count, start + count - 1))
+ return self._get_ranges(start, count, _SEEK_HOLE, _SEEK_DATA)
+
+
+# Below goes the FIEMAP ioctl implementation, which is not very readable
+# because it deals with the rather complex FIEMAP ioctl. To understand the
+# code, you need to know the FIEMAP interface, which is documented in the
+# "Documentation/filesystems/fiemap.txt" file in the Linux kernel sources.
+
+# Format string for 'struct fiemap'
+_FIEMAP_FORMAT = "=QQLLLL"
+# sizeof(struct fiemap)
+_FIEMAP_SIZE = struct.calcsize(_FIEMAP_FORMAT)
+# Format string for 'struct fiemap_extent'
+_FIEMAP_EXTENT_FORMAT = "=QQQQQLLLL"
+# sizeof(struct fiemap_extent)
+_FIEMAP_EXTENT_SIZE = struct.calcsize(_FIEMAP_EXTENT_FORMAT)
+# The FIEMAP ioctl number
+_FIEMAP_IOCTL = 0xC020660B
+# This FIEMAP ioctl flag which instructs the kernel to sync the file before
+# reading the block map
+_FIEMAP_FLAG_SYNC = 0x00000001
+# Size of the buffer for 'struct fiemap_extent' elements which will be used
+# when invoking the FIEMAP ioctl. The larger is the buffer, the less times the
+# FIEMAP ioctl will be invoked.
+_FIEMAP_BUFFER_SIZE = 256 * 1024
+
+class FilemapFiemap(_FilemapBase):
+ """
+ This class provides API to the FIEMAP ioctl. Namely, it allows to iterate
+ over all mapped blocks and over all holes.
+
+ This class synchronizes the image file every time it invokes the FIEMAP
+ ioctl in order to work-around early FIEMAP implementation kernel bugs.
+ """
+
+ def __init__(self, image, log=None):
+ """
+ Initialize a class instance. The 'image' argument is full the file
+ object to operate on.
+ """
+
+ # Call the base class constructor first
+ _FilemapBase.__init__(self, image, log)
+ self._log.debug("FilemapFiemap: initializing")
+
+ self._buf_size = _FIEMAP_BUFFER_SIZE
+
+ # Calculate how many 'struct fiemap_extent' elements fit the buffer
+ self._buf_size -= _FIEMAP_SIZE
+ self._fiemap_extent_cnt = self._buf_size // _FIEMAP_EXTENT_SIZE
+ assert self._fiemap_extent_cnt > 0
+ self._buf_size = self._fiemap_extent_cnt * _FIEMAP_EXTENT_SIZE
+ self._buf_size += _FIEMAP_SIZE
+
+ # Allocate a mutable buffer for the FIEMAP ioctl
+ self._buf = array.array('B', [0] * self._buf_size)
+
+ # Check if the FIEMAP ioctl is supported
+ self.block_is_mapped(0)
+
+ def _invoke_fiemap(self, block, count):
+ """
+ Invoke the FIEMAP ioctl for 'count' blocks of the file starting from
+ block number 'block'.
+
+ The full result of the operation is stored in 'self._buf' on exit.
+ Returns the unpacked 'struct fiemap' data structure in form of a python
+ list (just like 'struct.upack()').
+ """
+
+ if self.blocks_cnt != 0 and (block < 0 or block >= self.blocks_cnt):
+ raise Error("bad block number %d, should be within [0, %d]"
+ % (block, self.blocks_cnt))
+
+ # Initialize the 'struct fiemap' part of the buffer. We use the
+ # '_FIEMAP_FLAG_SYNC' flag in order to make sure the file is
+ # synchronized. The reason for this is that early FIEMAP
+ # implementations had many bugs related to cached dirty data, and
+ # synchronizing the file is a necessary work-around.
+ struct.pack_into(_FIEMAP_FORMAT, self._buf, 0, block * self.block_size,
+ count * self.block_size, _FIEMAP_FLAG_SYNC, 0,
+ self._fiemap_extent_cnt, 0)
+
+ try:
+ fcntl.ioctl(self._f_image, _FIEMAP_IOCTL, self._buf, 1)
+ except IOError as err:
+ # Note, the FIEMAP ioctl is supported by the Linux kernel starting
+ # from version 2.6.28 (year 2008).
+ if err.errno == os.errno.EOPNOTSUPP:
+ errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \
+ "by the file-system"
+ self._log.debug(errstr)
+ raise ErrorNotSupp(errstr)
+ if err.errno == os.errno.ENOTTY:
+ errstr = "FilemapFiemap: the FIEMAP ioctl is not supported " \
+ "by the kernel"
+ self._log.debug(errstr)
+ raise ErrorNotSupp(errstr)
+ raise Error("the FIEMAP ioctl failed for '%s': %s"
+ % (self._image_path, err))
+
+ return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE])
+
+ def block_is_mapped(self, block):
+ """Refer the '_FilemapBase' class for the documentation."""
+ struct_fiemap = self._invoke_fiemap(block, 1)
+
+ # The 3rd element of 'struct_fiemap' is the 'fm_mapped_extents' field.
+ # If it contains zero, the block is not mapped, otherwise it is
+ # mapped.
+ result = bool(struct_fiemap[3])
+ self._log.debug("FilemapFiemap: block_is_mapped(%d) returns %s"
+ % (block, result))
+ return result
+
+ def block_is_unmapped(self, block):
+ """Refer the '_FilemapBase' class for the documentation."""
+ return not self.block_is_mapped(block)
+
+ def _unpack_fiemap_extent(self, index):
+ """
+ Unpack a 'struct fiemap_extent' structure object number 'index' from
+ the internal 'self._buf' buffer.
+ """
+
+ offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index
+ return struct.unpack(_FIEMAP_EXTENT_FORMAT,
+ self._buf[offset : offset + _FIEMAP_EXTENT_SIZE])
+
+ def _do_get_mapped_ranges(self, start, count):
+ """
+ Implements most the functionality for the 'get_mapped_ranges()'
+ generator: invokes the FIEMAP ioctl, walks through the mapped extents
+ and yields mapped block ranges. However, the ranges may be consecutive
+ (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()' simply merges
+ them.
+ """
+
+ block = start
+ while block < start + count:
+ struct_fiemap = self._invoke_fiemap(block, count)
+
+ mapped_extents = struct_fiemap[3]
+ if mapped_extents == 0:
+ # No more mapped blocks
+ return
+
+ extent = 0
+ while extent < mapped_extents:
+ fiemap_extent = self._unpack_fiemap_extent(extent)
+
+ # Start of the extent
+ extent_start = fiemap_extent[0]
+ # Starting block number of the extent
+ extent_block = extent_start // self.block_size
+ # Length of the extent
+ extent_len = fiemap_extent[2]
+ # Count of blocks in the extent
+ extent_count = extent_len // self.block_size
+
+ # Extent length and offset have to be block-aligned
+ assert extent_start % self.block_size == 0
+ assert extent_len % self.block_size == 0
+
+ if extent_block > start + count - 1:
+ return
+
+ first = max(extent_block, block)
+ last = min(extent_block + extent_count, start + count) - 1
+ yield (first, last)
+
+ extent += 1
+
+ block = extent_block + extent_count
+
+ def get_mapped_ranges(self, start, count):
+ """Refer the '_FilemapBase' class for the documentation."""
+ self._log.debug("FilemapFiemap: get_mapped_ranges(%d, %d(%d))"
+ % (start, count, start + count - 1))
+ iterator = self._do_get_mapped_ranges(start, count)
+ first_prev, last_prev = next(iterator)
+
+ for first, last in iterator:
+ if last_prev == first - 1:
+ last_prev = last
+ else:
+ self._log.debug("FilemapFiemap: yielding range (%d, %d)"
+ % (first_prev, last_prev))
+ yield (first_prev, last_prev)
+ first_prev, last_prev = first, last
+
+ self._log.debug("FilemapFiemap: yielding range (%d, %d)"
+ % (first_prev, last_prev))
+ yield (first_prev, last_prev)
+
+ def get_unmapped_ranges(self, start, count):
+ """Refer the '_FilemapBase' class for the documentation."""
+ self._log.debug("FilemapFiemap: get_unmapped_ranges(%d, %d(%d))"
+ % (start, count, start + count - 1))
+ hole_first = start
+ for first, last in self._do_get_mapped_ranges(start, count):
+ if first > hole_first:
+ self._log.debug("FilemapFiemap: yielding range (%d, %d)"
+ % (hole_first, first - 1))
+ yield (hole_first, first - 1)
+
+ hole_first = last + 1
+
+ if hole_first < start + count:
+ self._log.debug("FilemapFiemap: yielding range (%d, %d)"
+ % (hole_first, start + count - 1))
+ yield (hole_first, start + count - 1)
+
+def filemap(image, log=None):
+ """
+ Create and return an instance of a Filemap class - 'FilemapFiemap' or
+ 'FilemapSeek', depending on what the system we run on supports. If the
+ FIEMAP ioctl is supported, an instance of the 'FilemapFiemap' class is
+ returned. Otherwise, if 'SEEK_HOLE' is supported an instance of the
+ 'FilemapSeek' class is returned. If none of these are supported, the
+ function generates an 'Error' type exception.
+ """
+
+ try:
+ return FilemapFiemap(image, log)
+ except ErrorNotSupp:
+ return FilemapSeek(image, log)
+
+def sparse_copy(src_fname, dst_fname, offset=0, skip=0, api=None):
+ """Efficiently copy sparse file to or into another file."""
+ if not api:
+ api = filemap
+ fmap = api(src_fname)
+ try:
+ dst_file = open(dst_fname, 'r+b')
+ except IOError:
+ dst_file = open(dst_fname, 'wb')
+ dst_file.truncate(os.path.getsize(src_fname))
+
+ for first, last in fmap.get_mapped_ranges(0, fmap.blocks_cnt):
+ start = first * fmap.block_size
+ end = (last + 1) * fmap.block_size
+
+ if start < skip < end:
+ fmap._f_image.seek(skip, os.SEEK_SET)
+ else:
+ fmap._f_image.seek(start, os.SEEK_SET)
+ dst_file.seek(offset + start, os.SEEK_SET)
+
+ chunk_size = 1024 * 1024
+ to_read = end - start
+ read = 0
+
+ while read < to_read:
+ if read + chunk_size > to_read:
+ chunk_size = to_read - read
+ chunk = fmap._f_image.read(chunk_size)
+ dst_file.write(chunk)
+ read += chunk_size
+ dst_file.close()
diff --git a/scripts/lib/image/help.py b/scripts/lib/wic/help.py
index e1eb265979..d6e027d253 100644
--- a/scripts/lib/image/help.py
+++ b/scripts/lib/wic/help.py
@@ -28,9 +28,12 @@
import subprocess
import logging
+from wic.pluginbase import PluginMgr, PLUGIN_TYPES
+
+logger = logging.getLogger('wic')
def subcommand_error(args):
- logging.info("invalid subcommand %s" % args[0])
+ logger.info("invalid subcommand %s", args[0])
def display_help(subcommand, subcommands):
@@ -40,9 +43,11 @@ def display_help(subcommand, subcommands):
if subcommand not in subcommands:
return False
- help = subcommands.get(subcommand, subcommand_error)[2]
+ hlp = subcommands.get(subcommand, subcommand_error)[2]
+ if callable(hlp):
+ hlp = hlp()
pager = subprocess.Popen('less', stdin=subprocess.PIPE)
- pager.communicate(help)
+ pager.communicate(hlp.encode('utf-8'))
return True
@@ -55,19 +60,38 @@ def wic_help(args, usage_str, subcommands):
print(usage_str)
+def get_wic_plugins_help():
+ """
+ Combine wic_plugins_help with the help for every known
+ source plugin.
+ """
+ result = wic_plugins_help
+ for plugin_type in PLUGIN_TYPES:
+ result += '\n\n%s PLUGINS\n\n' % plugin_type.upper()
+ for name, plugin in PluginMgr.get_plugins(plugin_type).items():
+ result += "\n %s plugin:\n" % name
+ if plugin.__doc__:
+ result += plugin.__doc__
+ else:
+ result += "\n %s is missing docstring\n" % plugin
+ return result
+
+
def invoke_subcommand(args, parser, main_command_usage, subcommands):
"""
Dispatch to subcommand handler borrowed from combo-layer.
Should use argparse, but has to work in 2.6.
"""
if not args:
- logging.error("No subcommand specified, exiting")
+ logger.error("No subcommand specified, exiting")
parser.print_help()
+ return 1
elif args[0] == "help":
wic_help(args, main_command_usage, subcommands)
elif args[0] not in subcommands:
- logging.error("Unsupported subcommand %s, exiting\n" % (args[0]))
+ logger.error("Unsupported subcommand %s, exiting\n", args[0])
parser.print_help()
+ return 1
else:
usage = subcommands.get(args[0], subcommand_error)[1]
subcommands.get(args[0], subcommand_error)[0](args[1:], usage)
@@ -81,19 +105,17 @@ wic_usage = """
Create a customized OpenEmbedded image
- usage: wic [--version] [--help] COMMAND [ARGS]
+ usage: wic [--version] | [--help] | [COMMAND [ARGS]]
Current 'wic' commands are:
+ help Show help for command or one of the topics (see below)
create Create a new OpenEmbedded image
- list List available values for options and image properties
+ list List available canned images and source plugins
Help topics:
- overview wic overview - General overview of wic
- plugins wic plugins - Overview and API
- kickstart wic kickstart - wic kickstart reference
-
- See 'wic help <COMMAND or HELP TOPIC>' for more information on a specific
- command or help topic.
+ overview wic overview - General overview of wic
+ plugins wic plugins - Overview and API
+ kickstart wic kickstart - wic kickstart reference
"""
wic_help_usage = """
@@ -111,7 +133,7 @@ wic_create_usage = """
[-i <JSON PROPERTY FILE> | --infile <JSON PROPERTY_FILE>]
[-e | --image-name] [-s, --skip-build-check] [-D, --debug]
[-r, --rootfs-dir] [-b, --bootimg-dir]
- [-k, --kernel-dir] [-n, --native-sysroot]
+ [-k, --kernel-dir] [-n, --native-sysroot] [-f, --build-rootfs]
This command creates an OpenEmbedded image based on the 'OE kickstart
commands' found in the <wks file>.
@@ -129,10 +151,10 @@ NAME
SYNOPSIS
wic create <wks file or image name> [-o <DIRNAME> | --outdir <DIRNAME>]
- [-i <JSON PROPERTY FILE> | --infile <JSON PROPERTY_FILE>]
[-e | --image-name] [-s, --skip-build-check] [-D, --debug]
[-r, --rootfs-dir] [-b, --bootimg-dir]
- [-k, --kernel-dir] [-n, --native-sysroot]
+ [-k, --kernel-dir] [-n, --native-sysroot] [-f, --build-rootfs]
+ [-c, --compress-with] [-m, --bmap]
DESCRIPTION
This command creates an OpenEmbedded image based on the 'OE
@@ -167,6 +189,8 @@ DESCRIPTION
The -n option is used to specify the path to the native sysroot
containing the tools to use to build the image.
+ The -f option is used to build rootfs by running "bitbake <image>"
+
The -s option is used to skip the build check. The build check is
a simple sanity check used to determine whether the user has
sourced the build environment so that the -e option can operate
@@ -197,32 +221,24 @@ DESCRIPTION
The -o option can be used to place the image in a directory with a
different name and location.
- As an alternative to the wks file, the image-specific properties
- that define the values that will be used to generate a particular
- image can be specified on the command-line using the -i option and
- supplying a JSON object consisting of the set of name:value pairs
- needed by image creation.
+ The -c option is used to specify compressor utility to compress
+ an image. gzip, bzip2 and xz compressors are supported.
- The set of properties available for a given image type can be
- listed using the 'wic list' command.
+ The -m option is used to produce .bmap file for the image. This file
+ can be used to flash image using bmaptool utility.
"""
wic_list_usage = """
- List available OpenEmbedded image properties and values
+ List available OpenEmbedded images and source plugins
usage: wic list images
wic list <image> help
wic list source-plugins
- wic list properties
- wic list properties <wks file>
- wic list property <property>
- [-o <JSON PROPERTY FILE> | --outfile <JSON PROPERTY_FILE>]
This command enumerates the set of available canned images as well as
- help for those images. It also can be used to enumerate the complete
- set of possible values for a specified option or property needed by
- the image creation process.
+ help for those images. It also can be used to list of available source
+ plugins.
The first form enumerates all the available 'canned' images.
@@ -232,44 +248,27 @@ wic_list_usage = """
The third form enumerates all the available --sources (source
plugins).
- The fourth form enumerates all the possible values that exist and can
- be specified in an OE kickstart (wks) file.
-
- The fifth form enumerates all the possible options that exist for the
- set of properties specified in a given OE kickstart (ks) file.
-
- The final form enumerates all the possible values that exist and can
- be specified for any given OE kickstart (wks) property.
-
See 'wic help list' for more details.
"""
wic_list_help = """
NAME
- wic list - List available OpenEmbedded image properties and values
+ wic list - List available OpenEmbedded images and source plugins
SYNOPSIS
wic list images
wic list <image> help
wic list source-plugins
- wic list properties
- wic list properties <wks file>
- wic list property <property>
- [-o <JSON PROPERTY FILE> | --outfile <JSON PROPERTY_FILE>]
DESCRIPTION
- This command enumerates the complete set of possible values for a
- specified option or property needed by the image creation process.
-
This command enumerates the set of available canned images as well
- as help for those images. It also can be used to enumerate the
- complete set of possible values for a specified option or property
- needed by the image creation process.
+ as help for those images. It also can be used to list available
+ source plugins.
The first form enumerates all the available 'canned' images.
These are actually just the set of .wks files that have been moved
- into the /scripts/lib/image/canned-wks directory).
+ into the /scripts/lib/wic/canned-wks directory).
The second form lists the detailed help information for a specific
'canned' image.
@@ -282,60 +281,6 @@ DESCRIPTION
sources listed by the 'list source-plugins' command. Users can
also add their own source plugins - see 'wic help plugins' for
details.
-
- The third form enumerates all the possible values that exist and
- can be specified in a OE kickstart (wks) file. The output of this
- can be used by the third form to print the description and
- possible values of a specific property.
-
- The fourth form enumerates all the possible options that exist for
- the set of properties specified in a given OE kickstart (wks)
- file. If the -o option is specified, the list of properties, in
- addition to being displayed, will be written to the specified file
- as a JSON object. In this case, the object will consist of the
- set of name:value pairs corresponding to the (possibly nested)
- dictionary of properties defined by the input statements used by
- the image. Some example output for the 'list <wks file>' command:
-
- $ wic list test.ks
- "part" : {
- "mountpoint" : "/"
- "fstype" : "ext3"
- }
- "part" : {
- "mountpoint" : "/home"
- "fstype" : "ext3"
- "offset" : "10000"
- }
- "bootloader" : {
- "type" : "efi"
- }
- .
- .
- .
-
- Each entry in the output consists of the name of the input element
- e.g. "part", followed by the properties defined for that
- element enclosed in braces. This information should provide
- sufficient information to create a complete user interface with.
-
- The final form enumerates all the possible values that exist and
- can be specified for any given OE kickstart (wks) property. If
- the -o option is specified, the list of values for the given
- property, in addition to being displayed, will be written to the
- specified file as a JSON object. In this case, the object will
- consist of the set of name:value pairs corresponding to the array
- of property values associated with the property.
-
- $ wic list property part
- ["mountpoint", "where the partition should be mounted"]
- ["fstype", "filesytem type of the partition"]
- ["ext3"]
- ["ext4"]
- ["btrfs"]
- ["swap"]
- ["offset", "offset of the partition within the image"]
-
"""
wic_plugins_help = """
@@ -428,12 +373,7 @@ DESCRIPTION
This scheme is extensible - adding more hooks is a simple matter
of adding more plugin methods to SourcePlugin and derived classes.
- The code that then needs to call the plugin methods uses
- plugin.get_source_plugin_methods() to find the method(s) needed by
- the call; this is done by filling up a dict with keys containing
- the method names of interest - on success, these will be filled in
- with the actual methods. Please see the implementation for
- examples and details.
+ Please see the implementation for details.
"""
wic_overview_help = """
@@ -528,8 +468,8 @@ DESCRIPTION
usage: wic create <wks file or image name> [-o <DIRNAME> | ...]
[-i <JSON PROPERTY FILE> | --infile <JSON PROPERTY_FILE>]
[-e | --image-name] [-s, --skip-build-check] [-D, --debug]
- [-r, --rootfs-dir] [-b, --bootimg-dir]
- [-k, --kernel-dir] [-n, --native-sysroot]
+ [-r, --rootfs-dir] [-b, --bootimg-dir] [-k, --kernel-dir]
+ [-n, --native-sysroot] [-f, --build-rootfs]
This command creates an OpenEmbedded image based on the 'OE
kickstart commands' found in the <wks file>.
@@ -572,7 +512,7 @@ DESCRIPTION
NATIVE_SYSROOT: ...
The image(s) were created using OE kickstart file:
- .../scripts/lib/image/canned-wks/directdisk.wks
+ .../scripts/lib/wic/canned-wks/directdisk.wks
The output shows the name and location of the image created, and
so that you know exactly what was used to generate the image, each
@@ -599,18 +539,33 @@ DESCRIPTION
and also uses the -o option to have wic create the output
somewhere other than the default /var/tmp/wic:
- $ wic create ~/test.wks -o /home/trz/testwic --rootfs-dir
- /home/trz/yocto/build/tmp/work/crownbay/core-image-minimal/1.0-r0/rootfs
- --bootimg-dir /home/trz/yocto/build/tmp/sysroots/crownbay/usr/share
- --kernel-dir /home/trz/yocto/build/tmp/sysroots/crownbay/usr/src/kernel
- --native-sysroot /home/trz/yocto/build/tmp/sysroots/x86_64-linux
+ $ wic create ./test.wks -o ./out --rootfs-dir
+ tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs
+ --bootimg-dir tmp/sysroots/qemux86-64/usr/share
+ --kernel-dir tmp/deploy/images/qemux86-64
+ --native-sysroot tmp/sysroots/x86_64-linux
- Creating image(s)...
+ Creating image(s)...
- Info: The new image(s) can be found here:
- /home/trz/testwic/build/test-201309260032-sda.direct
+ Info: The new image(s) can be found here:
+ out/build/test-201507211313-sda.direct
+
+ The following build artifacts were used to create the image(s):
+ ROOTFS_DIR: tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/rootfs
+ BOOTIMG_DIR: tmp/sysroots/qemux86-64/usr/share
+ KERNEL_DIR: tmp/deploy/images/qemux86-64
+ NATIVE_SYSROOT: tmp/sysroots/x86_64-linux
+
+ The image(s) were created using OE kickstart file:
+ ./test.wks
+
+ Here is a content of test.wks:
+
+ part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
+ part / --source rootfs --ondisk sda --fstype=ext3 --label platform --align 1024
+
+ bootloader --timeout=0 --append="rootwait rootfstype=ext3 video=vesafb vga=0x318 console=tty0"
- ...
Finally, here's an example of the actual partition language
commands used to generate the mkefidisk image i.e. these are the
@@ -662,7 +617,7 @@ DESCRIPTION
This command creates a partition on the system and uses the
following syntax:
- part <mountpoint>
+ part [<mountpoint>]
The <mountpoint> is where the partition will be mounted and
must take of one of the following forms:
@@ -671,6 +626,16 @@ DESCRIPTION
swap: The partition will be used as swap space.
+ If a <mountpoint> is not specified the partition will be created
+ but will not be mounted.
+
+ Partitions with a <mountpoint> specified will be automatically mounted.
+ This is achieved by wic adding entries to the fstab during image
+ generation. In order for a valid fstab to be generated one of the
+ --ondrive, --ondisk or --use-uuid partition options must be used for
+ each partition that specifies a mountpoint.
+
+
The following are supported 'part' options:
--size: The minimum partition size. Specify an integer value
@@ -678,6 +643,12 @@ DESCRIPTION
not specified, the size is in MB.
You do not need this option if you use --source.
+ --fixed-size: Exact partition size. Value format is the same
+ as for --size option. This option cannot be
+ specified along with --size. If partition data
+ is larger than --fixed-size and error will be
+ raised when assembling disk image.
+
--source: This option is a wic-specific option that names the
source of the data that will populate the
partition. The most common value for this option
@@ -705,6 +676,10 @@ DESCRIPTION
up being are dependent on the given plugin
implementation.
+ If --source option is not used, the wic command
+ will create empty partition. --size parameter has
+ to be used to specify size of empty partition.
+
--ondisk or --ondrive: Forces the partition to be created on
a particular disk.
@@ -712,6 +687,8 @@ DESCRIPTION
apply to partitions created using '--source rootfs' (see
--source above). Valid values are:
+ vfat
+ msdos
ext2
ext3
ext4
@@ -743,17 +720,43 @@ DESCRIPTION
partition table. It may be useful for
bootloaders.
+ --exclude-path: This option is specific to wic. It excludes the given
+ relative path from the resulting image. If the path
+ ends with a slash, only the content of the directory
+ is omitted, not the directory itself. This option only
+ has an effect with the rootfs source plugin.
+
--extra-space: This option is specific to wic. It adds extra
space after the space filled by the content
of the partition. The final size can go
beyond the size specified by --size.
- By default, 10MB.
+ By default, 10MB. This option cannot be used
+ with --fixed-size option.
--overhead-factor: This option is specific to wic. The
size of the partition is multiplied by
this factor. It has to be greater than or
- equal to 1.
- The default value is 1.3.
+ equal to 1. The default value is 1.3.
+ This option cannot be used with --fixed-size
+ option.
+
+ --part-type: This option is specific to wic. It specifies partition
+ type GUID for GPT partitions.
+ List of partition type GUIDS can be found here:
+ http://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_type_GUIDs
+
+ --use-uuid: This option is specific to wic. It makes wic to generate
+ random globally unique identifier (GUID) for the partition
+ and use it in bootloader configuration to specify root partition.
+
+ --uuid: This option is specific to wic. It specifies partition UUID.
+ It's useful if preconfigured partition UUID is added to kernel command line
+ in bootloader configuration before running wic. In this case .wks file can
+ be generated or modified to set preconfigured parition UUID using this option.
+
+ --system-id: This option is specific to wic. It specifies partition system id. It's useful
+ for the harware that requires non-default partition system ids. The parameter
+ in one byte long hex number either with 0x prefix or without it.
* bootloader
@@ -767,8 +770,28 @@ DESCRIPTION
bootloader command-line - for example, the syslinux
APPEND or grub kernel command line.
+ --configfile: Specifies a user defined configuration file for
+ the bootloader. This file must be located in the
+ canned-wks folder or could be the full path to the
+ file. Using this option will override any other
+ bootloader option.
+
Note that bootloader functionality and boot partitions are
implemented by the various --source plugins that implement
bootloader functionality; the bootloader command essentially
provides a means of modifying bootloader configuration.
+
+ * include
+
+ This command allows the user to include the content of .wks file
+ into original .wks file.
+
+ Command uses the following syntax:
+
+ include <file>
+
+ The <file> is either path to the file or its name. If name is
+ specified wic will try to find file in the directories with canned
+ .wks files.
+
"""
diff --git a/scripts/lib/wic/imager/__init__.py b/scripts/lib/wic/imager/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
--- a/scripts/lib/wic/imager/__init__.py
+++ /dev/null
diff --git a/scripts/lib/wic/imager/baseimager.py b/scripts/lib/wic/imager/baseimager.py
deleted file mode 100644
index e8305272a2..0000000000
--- a/scripts/lib/wic/imager/baseimager.py
+++ /dev/null
@@ -1,193 +0,0 @@
-#!/usr/bin/env python -tt
-#
-# Copyright (c) 2007 Red Hat Inc.
-# Copyright (c) 2009, 2010, 2011 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-from __future__ import with_statement
-import os, sys
-import tempfile
-import shutil
-
-from wic import kickstart
-from wic import msger
-from wic.utils.errors import CreatorError
-from wic.utils import misc, runner, fs_related as fs
-
-class BaseImageCreator(object):
- """Base class for image creation.
-
- BaseImageCreator is the simplest creator class available; it will
- create a system image according to the supplied kickstart file.
-
- e.g.
-
- import wic.imgcreate as imgcreate
- ks = imgcreate.read_kickstart("foo.ks")
- imgcreate.ImageCreator(ks, "foo").create()
- """
-
- def __del__(self):
- self.cleanup()
-
- def __init__(self, createopts = None):
- """Initialize an ImageCreator instance.
-
- ks -- a pykickstart.KickstartParser instance; this instance will be
- used to drive the install by e.g. providing the list of packages
- to be installed, the system configuration and %post scripts
-
- name -- a name for the image; used for e.g. image filenames or
- filesystem labels
- """
-
- self.__builddir = None
-
- self.ks = None
- self.name = "target"
- self.tmpdir = "/var/tmp/wic"
- self.workdir = "/var/tmp/wic/build"
-
- # setup tmpfs tmpdir when enabletmpfs is True
- self.enabletmpfs = False
-
- if createopts:
- # Mapping table for variables that have different names.
- optmap = {"outdir" : "destdir",
- }
-
- # update setting from createopts
- for key in createopts.keys():
- if key in optmap:
- option = optmap[key]
- else:
- option = key
- setattr(self, option, createopts[key])
-
- self.destdir = os.path.abspath(os.path.expanduser(self.destdir))
-
- self._dep_checks = ["ls", "bash", "cp", "echo"]
-
- # Output image file names
- self.outimage = []
-
- # No ks provided when called by convertor, so skip the dependency check
- if self.ks:
- # If we have btrfs partition we need to check necessary tools
- for part in self.ks.handler.partition.partitions:
- if part.fstype and part.fstype == "btrfs":
- self._dep_checks.append("mkfs.btrfs")
- break
-
- # make sure the specified tmpdir and cachedir exist
- if not os.path.exists(self.tmpdir):
- os.makedirs(self.tmpdir)
-
-
- #
- # Hooks for subclasses
- #
- def _create(self):
- """Create partitions for the disk image(s)
-
- This is the hook where subclasses may create the partitions
- that will be assembled into disk image(s).
-
- There is no default implementation.
- """
- pass
-
- def _cleanup(self):
- """Undo anything performed in _create().
-
- This is the hook where subclasses must undo anything which was
- done in _create().
-
- There is no default implementation.
-
- """
- pass
-
- #
- # Actual implementation
- #
- def __ensure_builddir(self):
- if not self.__builddir is None:
- return
-
- try:
- self.workdir = os.path.join(self.tmpdir, "build")
- if not os.path.exists(self.workdir):
- os.makedirs(self.workdir)
- self.__builddir = tempfile.mkdtemp(dir = self.workdir,
- prefix = "imgcreate-")
- except OSError, (err, msg):
- raise CreatorError("Failed create build directory in %s: %s" %
- (self.tmpdir, msg))
-
- def __setup_tmpdir(self):
- if not self.enabletmpfs:
- return
-
- runner.show('mount -t tmpfs -o size=4G tmpfs %s' % self.workdir)
-
- def __clean_tmpdir(self):
- if not self.enabletmpfs:
- return
-
- runner.show('umount -l %s' % self.workdir)
-
- def create(self):
- """Create partitions for the disk image(s)
-
- Create the partitions that will be assembled into disk
- image(s).
- """
- self.__setup_tmpdir()
- self.__ensure_builddir()
-
- self._create()
-
- def cleanup(self):
- """Undo anything performed in create().
-
- Note, make sure to call this method once finished with the creator
- instance in order to ensure no stale files are left on the host e.g.:
-
- creator = ImageCreator(ks, name)
- try:
- creator.create()
- finally:
- creator.cleanup()
-
- """
- if not self.__builddir:
- return
-
- self._cleanup()
-
- shutil.rmtree(self.__builddir, ignore_errors = True)
- self.__builddir = None
-
- self.__clean_tmpdir()
-
-
- def print_outimage_info(self):
- msg = "The new image can be found here:\n"
- self.outimage.sort()
- for file in self.outimage:
- msg += ' %s\n' % os.path.abspath(file)
-
- msger.info(msg)
diff --git a/scripts/lib/wic/imager/direct.py b/scripts/lib/wic/imager/direct.py
deleted file mode 100644
index d368401af4..0000000000
--- a/scripts/lib/wic/imager/direct.py
+++ /dev/null
@@ -1,379 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (c) 2013, Intel Corporation.
-# All rights reserved.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# DESCRIPTION
-# This implements the 'direct' image creator class for 'wic'
-#
-# AUTHORS
-# Tom Zanussi <tom.zanussi (at] linux.intel.com>
-#
-
-import os
-import stat
-import shutil
-
-from wic import kickstart, msger
-from wic.utils import fs_related, runner, misc
-from wic.utils.partitionedfs import Image
-from wic.utils.errors import CreatorError, ImageError
-from wic.imager.baseimager import BaseImageCreator
-from wic.utils.oe.misc import *
-from wic.plugin import pluginmgr
-
-disk_methods = {
- "do_install_disk":None,
-}
-
-class DirectImageCreator(BaseImageCreator):
- """
- Installs a system into a file containing a partitioned disk image.
-
- DirectImageCreator is an advanced ImageCreator subclass; an image
- file is formatted with a partition table, each partition created
- from a rootfs or other OpenEmbedded build artifact and dd'ed into
- the virtual disk. The disk image can subsequently be dd'ed onto
- media and used on actual hardware.
- """
-
- def __init__(self, oe_builddir, image_output_dir, rootfs_dir, bootimg_dir,
- kernel_dir, native_sysroot, creatoropts=None):
- """
- Initialize a DirectImageCreator instance.
-
- This method takes the same arguments as ImageCreator.__init__()
- """
- BaseImageCreator.__init__(self, creatoropts)
-
- self.__image = None
- self.__disks = {}
- self.__disk_format = "direct"
- self._disk_names = []
- self._ptable_format = self.ks.handler.bootloader.ptable
-
- self.oe_builddir = oe_builddir
- if image_output_dir:
- self.tmpdir = image_output_dir
- self.rootfs_dir = rootfs_dir
- self.bootimg_dir = bootimg_dir
- self.kernel_dir = kernel_dir
- self.native_sysroot = native_sysroot
-
- def __get_part_num(self, num, parts):
- """calculate the real partition number, accounting for partitions not
- in the partition table and logical partitions
- """
- realnum = 0
- for n, p in enumerate(parts, 1):
- if not p.no_table:
- realnum += 1
- if n == num:
- if p.no_table:
- return 0
- if self._ptable_format == 'msdos' and realnum > 3:
- # account for logical partition numbering, ex. sda5..
- return realnum + 1
- return realnum
-
- def __write_fstab(self, image_rootfs):
- """overriden to generate fstab (temporarily) in rootfs. This is called
- from _create, make sure it doesn't get called from
- BaseImage.create()
- """
- if image_rootfs is None:
- return None
-
- fstab = image_rootfs + "/etc/fstab"
- if not os.path.isfile(fstab):
- return None
-
- parts = self._get_parts()
-
- self._save_fstab(fstab)
- fstab_lines = self._get_fstab(fstab, parts)
- self._update_fstab(fstab_lines, parts)
- self._write_fstab(fstab, fstab_lines)
-
- return fstab
-
- def _update_fstab(self, fstab_lines, parts):
- """Assume partition order same as in wks"""
- for num, p in enumerate(parts, 1):
- pnum = self.__get_part_num(num, parts)
- if not p.mountpoint or p.mountpoint == "/" or p.mountpoint == "/boot" or pnum == 0:
- continue
-
- part = ''
- # mmc device partitions are named mmcblk0p1, mmcblk0p2..
- if p.disk.startswith('mmcblk'):
- part = 'p'
-
- device_name = "/dev/" + p.disk + part + str(pnum)
-
- opts = "defaults"
- if p.fsopts:
- opts = p.fsopts
-
- fstab_entry = device_name + "\t" + \
- p.mountpoint + "\t" + \
- p.fstype + "\t" + \
- opts + "\t0\t0\n"
- fstab_lines.append(fstab_entry)
-
- def _write_fstab(self, fstab, fstab_lines):
- fstab = open(fstab, "w")
- for line in fstab_lines:
- fstab.write(line)
- fstab.close()
-
- def _save_fstab(self, fstab):
- """Save the current fstab in rootfs"""
- shutil.copyfile(fstab, fstab + ".orig")
-
- def _restore_fstab(self, fstab):
- """Restore the saved fstab in rootfs"""
- if fstab is None:
- return
- shutil.move(fstab + ".orig", fstab)
-
- def _get_fstab(self, fstab, parts):
- """Return the desired contents of /etc/fstab."""
- f = open(fstab, "r")
- fstab_contents = f.readlines()
- f.close()
-
- return fstab_contents
-
- def set_bootimg_dir(self, bootimg_dir):
- """
- Accessor for bootimg_dir, the actual location used for the source
- of the bootimg. Should be set by source plugins (only if they
- change the default bootimg source) so the correct info gets
- displayed for print_outimage_info().
- """
- self.bootimg_dir = bootimg_dir
-
- def _get_parts(self):
- if not self.ks:
- raise CreatorError("Failed to get partition info, "
- "please check your kickstart setting.")
-
- # Set a default partition if no partition is given out
- if not self.ks.handler.partition.partitions:
- partstr = "part / --size 1900 --ondisk sda --fstype=ext3"
- args = partstr.split()
- pd = self.ks.handler.partition.parse(args[1:])
- if pd not in self.ks.handler.partition.partitions:
- self.ks.handler.partition.partitions.append(pd)
-
- # partitions list from kickstart file
- return kickstart.get_partitions(self.ks)
-
- def get_disk_names(self):
- """ Returns a list of physical target disk names (e.g., 'sdb') which
- will be created. """
-
- if self._disk_names:
- return self._disk_names
-
- #get partition info from ks handler
- parts = self._get_parts()
-
- for i in range(len(parts)):
- if parts[i].disk:
- disk_name = parts[i].disk
- else:
- raise CreatorError("Failed to create disks, no --ondisk "
- "specified in partition line of ks file")
-
- if parts[i].mountpoint and not parts[i].fstype:
- raise CreatorError("Failed to create disks, no --fstype "
- "specified for partition with mountpoint "
- "'%s' in the ks file")
-
- self._disk_names.append(disk_name)
-
- return self._disk_names
-
- def _full_name(self, name, extention):
- """ Construct full file name for a file we generate. """
- return "%s-%s.%s" % (self.name, name, extention)
-
- def _full_path(self, path, name, extention):
- """ Construct full file path to a file we generate. """
- return os.path.join(path, self._full_name(name, extention))
-
- def get_default_source_plugin(self):
- """
- The default source plugin i.e. the plugin that's consulted for
- overall image generation tasks outside of any particular
- partition. For convenience, we just hang it off the
- bootloader handler since it's the one non-partition object in
- any setup. By default the default plugin is set to the same
- plugin as the /boot partition; since we hang it off the
- bootloader object, the default can be explicitly set using the
- --source bootloader param.
- """
- return self.ks.handler.bootloader.source
-
- #
- # Actual implemention
- #
- def _create(self):
- """
- For 'wic', we already have our build artifacts - we just create
- filesystems from the artifacts directly and combine them into
- a partitioned image.
- """
- parts = self._get_parts()
-
- self.__image = Image()
-
- for p in parts:
- # as a convenience, set source to the boot partition source
- # instead of forcing it to be set via bootloader --source
- if not self.ks.handler.bootloader.source and p.mountpoint == "/boot":
- self.ks.handler.bootloader.source = p.source
-
- fstab = self.__write_fstab(self.rootfs_dir.get("ROOTFS_DIR"))
-
- for p in parts:
- # need to create the filesystems in order to get their
- # sizes before we can add them and do the layout.
- # Image.create() actually calls __format_disks() to create
- # the disk images and carve out the partitions, then
- # self.assemble() calls Image.assemble() which calls
- # __write_partitition() for each partition to dd the fs
- # into the partitions.
- p.prepare(self, self.workdir, self.oe_builddir, self.rootfs_dir,
- self.bootimg_dir, self.kernel_dir, self.native_sysroot)
-
-
- self.__image.add_partition(int(p.size),
- p.disk,
- p.mountpoint,
- p.source_file,
- p.fstype,
- p.label,
- fsopts = p.fsopts,
- boot = p.active,
- align = p.align,
- no_table = p.no_table,
- part_type = p.part_type)
-
- self._restore_fstab(fstab)
-
- self.__image.layout_partitions(self._ptable_format)
-
- self.__imgdir = self.workdir
- for disk_name, disk in self.__image.disks.items():
- full_path = self._full_path(self.__imgdir, disk_name, "direct")
- msger.debug("Adding disk %s as %s with size %s bytes" \
- % (disk_name, full_path, disk['min_size']))
- disk_obj = fs_related.DiskImage(full_path, disk['min_size'])
- self.__disks[disk_name] = disk_obj
- self.__image.add_disk(disk_name, disk_obj)
-
- self.__image.create()
-
- def assemble(self):
- """
- Assemble partitions into disk image(s)
- """
- for disk_name, disk in self.__image.disks.items():
- full_path = self._full_path(self.__imgdir, disk_name, "direct")
- msger.debug("Assembling disk %s as %s with size %s bytes" \
- % (disk_name, full_path, disk['min_size']))
- self.__image.assemble(full_path)
-
- def finalize(self):
- """
- Finalize the disk image.
-
- For example, prepare the image to be bootable by e.g.
- creating and installing a bootloader configuration.
-
- """
- source_plugin = self.get_default_source_plugin()
- if source_plugin:
- self._source_methods = pluginmgr.get_source_plugin_methods(source_plugin, disk_methods)
- for disk_name, disk in self.__image.disks.items():
- self._source_methods["do_install_disk"](disk, disk_name, self,
- self.workdir,
- self.oe_builddir,
- self.bootimg_dir,
- self.kernel_dir,
- self.native_sysroot)
-
- def print_outimage_info(self):
- """
- Print the image(s) and artifacts used, for the user.
- """
- msg = "The new image(s) can be found here:\n"
-
- parts = self._get_parts()
-
- for disk_name, disk in self.__image.disks.items():
- full_path = self._full_path(self.__imgdir, disk_name, "direct")
- msg += ' %s\n\n' % full_path
-
- msg += 'The following build artifacts were used to create the image(s):\n'
- for p in parts:
- if p.get_rootfs() is None:
- continue
- if p.mountpoint == '/':
- str = ':'
- else:
- str = '["%s"]:' % p.label
- msg += ' ROOTFS_DIR%s%s\n' % (str.ljust(20), p.get_rootfs())
-
- msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir
- msg += ' KERNEL_DIR: %s\n' % self.kernel_dir
- msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot
-
- msger.info(msg)
-
- def _get_boot_config(self):
- """
- Return the rootdev/root_part_uuid (if specified by
- --part-type)
-
- Assume partition order same as in wks
- """
- rootdev = None
- root_part_uuid = None
- parts = self._get_parts()
- for num, p in enumerate(parts, 1):
- if p.mountpoint == "/":
- part = ''
- if p.disk.startswith('mmcblk'):
- part = 'p'
-
- pnum = self.__get_part_num(num, parts)
- rootdev = "/dev/%s%s%-d" % (p.disk, part, pnum)
- root_part_uuid = p.part_type
-
- return (rootdev, root_part_uuid)
-
- def _cleanup(self):
- if not self.__image is None:
- try:
- self.__image.cleanup()
- except ImageError, err:
- msger.warning("%s" % err)
-
diff --git a/scripts/lib/wic/kickstart/__init__.py b/scripts/lib/wic/kickstart/__init__.py
deleted file mode 100644
index 10959213d1..0000000000
--- a/scripts/lib/wic/kickstart/__init__.py
+++ /dev/null
@@ -1,122 +0,0 @@
-#!/usr/bin/env python -tt
-#
-# Copyright (c) 2007 Red Hat, Inc.
-# Copyright (c) 2009, 2010, 2011 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-import os, sys, re
-import shutil
-import subprocess
-import string
-
-import pykickstart.sections as kssections
-import pykickstart.commands as kscommands
-import pykickstart.constants as ksconstants
-import pykickstart.errors as kserrors
-import pykickstart.parser as ksparser
-import pykickstart.version as ksversion
-from pykickstart.handlers.control import commandMap
-from pykickstart.handlers.control import dataMap
-
-from wic import msger
-from wic.utils import errors, misc, runner, fs_related as fs
-from custom_commands import wicboot, partition
-
-def read_kickstart(path):
- """Parse a kickstart file and return a KickstartParser instance.
-
- This is a simple utility function which takes a path to a kickstart file,
- parses it and returns a pykickstart KickstartParser instance which can
- be then passed to an ImageCreator constructor.
-
- If an error occurs, a CreatorError exception is thrown.
- """
-
- #version = ksversion.makeVersion()
- #ks = ksparser.KickstartParser(version)
-
- using_version = ksversion.DEVEL
- commandMap[using_version]["bootloader"] = wicboot.Wic_Bootloader
- commandMap[using_version]["part"] = partition.Wic_Partition
- commandMap[using_version]["partition"] = partition.Wic_Partition
- dataMap[using_version]["PartData"] = partition.Wic_PartData
- superclass = ksversion.returnClassForVersion(version=using_version)
-
- class KSHandlers(superclass):
- def __init__(self):
- superclass.__init__(self, mapping=commandMap[using_version])
-
- ks = ksparser.KickstartParser(KSHandlers(), errorsAreFatal=True)
-
- try:
- ks.readKickstart(path)
- except (kserrors.KickstartParseError, kserrors.KickstartError), err:
- msger.warning("Errors occurred when parsing kickstart file: %s\n" % path)
- msger.error("%s" % err)
-
- return ks
-
-def get_image_size(ks, default = None):
- __size = 0
- for p in ks.handler.partition.partitions:
- if p.mountpoint == "/" and p.size:
- __size = p.size
- if __size > 0:
- return int(__size) * 1024L
- else:
- return default
-
-def get_image_fstype(ks, default = None):
- for p in ks.handler.partition.partitions:
- if p.mountpoint == "/" and p.fstype:
- return p.fstype
- return default
-
-def get_image_fsopts(ks, default = None):
- for p in ks.handler.partition.partitions:
- if p.mountpoint == "/" and p.fsopts:
- return p.fsopts
- return default
-
-def get_timeout(ks, default = None):
- if not hasattr(ks.handler.bootloader, "timeout"):
- return default
- if ks.handler.bootloader.timeout is None:
- return default
- return int(ks.handler.bootloader.timeout)
-
-def get_kernel_args(ks, default = "ro rd.live.image"):
- if not hasattr(ks.handler.bootloader, "appendLine"):
- return default
- if ks.handler.bootloader.appendLine is None:
- return default
- return "%s %s" %(default, ks.handler.bootloader.appendLine)
-
-def get_menu_args(ks, default = ""):
- if not hasattr(ks.handler.bootloader, "menus"):
- return default
- if ks.handler.bootloader.menus in (None, ""):
- return default
- return "%s" % ks.handler.bootloader.menus
-
-def get_default_kernel(ks, default = None):
- if not hasattr(ks.handler.bootloader, "default"):
- return default
- if not ks.handler.bootloader.default:
- return default
- return ks.handler.bootloader.default
-
-def get_partitions(ks):
- return ks.handler.partition.partitions
diff --git a/scripts/lib/wic/kickstart/custom_commands/__init__.py b/scripts/lib/wic/kickstart/custom_commands/__init__.py
deleted file mode 100644
index f84c6d9e00..0000000000
--- a/scripts/lib/wic/kickstart/custom_commands/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from micpartition import Mic_Partition
-from micpartition import Mic_PartData
-from partition import Wic_Partition
-
-__all__ = (
- "Mic_Partition",
- "Mic_PartData",
- "Wic_Partition",
- "Wic_PartData",
-)
diff --git a/scripts/lib/wic/kickstart/custom_commands/micboot.py b/scripts/lib/wic/kickstart/custom_commands/micboot.py
deleted file mode 100644
index d162142506..0000000000
--- a/scripts/lib/wic/kickstart/custom_commands/micboot.py
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env python -tt
-#
-# Copyright (c) 2008, 2009, 2010 Intel, Inc.
-#
-# Anas Nashif
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-from pykickstart.base import *
-from pykickstart.errors import *
-from pykickstart.options import *
-from pykickstart.commands.bootloader import *
-
-class Mic_Bootloader(F8_Bootloader):
- def __init__(self, writePriority=10, appendLine="", driveorder=None,
- forceLBA=False, location="", md5pass="", password="",
- upgrade=False, menus=""):
- F8_Bootloader.__init__(self, writePriority, appendLine, driveorder,
- forceLBA, location, md5pass, password, upgrade)
-
- self.menus = ""
- self.ptable = "msdos"
-
- def _getArgsAsStr(self):
- ret = F8_Bootloader._getArgsAsStr(self)
-
- if self.menus == "":
- ret += " --menus=%s" %(self.menus,)
- if self.ptable:
- ret += " --ptable=\"%s\"" %(self.ptable,)
- return ret
-
- def _getParser(self):
- op = F8_Bootloader._getParser(self)
- op.add_option("--menus", dest="menus")
- op.add_option("--ptable", dest="ptable", type="string")
- return op
-
diff --git a/scripts/lib/wic/kickstart/custom_commands/micpartition.py b/scripts/lib/wic/kickstart/custom_commands/micpartition.py
deleted file mode 100644
index d6be008ceb..0000000000
--- a/scripts/lib/wic/kickstart/custom_commands/micpartition.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env python -tt
-#
-# Marko Saukko <marko.saukko@cybercom.com>
-#
-# Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
-#
-# This copyrighted material is made available to anyone wishing to use, modify,
-# copy, or redistribute it subject to the terms and conditions of the GNU
-# General Public License v.2. This program is distributed in the hope that it
-# will be useful, but WITHOUT ANY WARRANTY expressed or implied, including the
-# implied warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
-
-from pykickstart.commands.partition import *
-
-class Mic_PartData(FC4_PartData):
- removedKeywords = FC4_PartData.removedKeywords
- removedAttrs = FC4_PartData.removedAttrs
-
- def __init__(self, *args, **kwargs):
- FC4_PartData.__init__(self, *args, **kwargs)
- self.deleteRemovedAttrs()
- self.align = kwargs.get("align", None)
- self.extopts = kwargs.get("extopts", None)
- self.part_type = kwargs.get("part_type", None)
-
- def _getArgsAsStr(self):
- retval = FC4_PartData._getArgsAsStr(self)
-
- if self.align:
- retval += " --align=%d" % self.align
- if self.extopts:
- retval += " --extoptions=%s" % self.extopts
- if self.part_type:
- retval += " --part-type=%s" % self.part_type
-
- return retval
-
-class Mic_Partition(FC4_Partition):
- removedKeywords = FC4_Partition.removedKeywords
- removedAttrs = FC4_Partition.removedAttrs
-
- def _getParser(self):
- op = FC4_Partition._getParser(self)
- # The alignment value is given in kBytes. e.g., value 8 means that
- # the partition is aligned to start from 8096 byte boundary.
- op.add_option("--align", type="int", action="store", dest="align",
- default=None)
- op.add_option("--extoptions", type="string", action="store", dest="extopts",
- default=None)
- op.add_option("--part-type", type="string", action="store", dest="part_type",
- default=None)
- return op
diff --git a/scripts/lib/wic/kickstart/custom_commands/partition.py b/scripts/lib/wic/kickstart/custom_commands/partition.py
deleted file mode 100644
index 7cfeec4dd2..0000000000
--- a/scripts/lib/wic/kickstart/custom_commands/partition.py
+++ /dev/null
@@ -1,576 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (c) 2013, Intel Corporation.
-# All rights reserved.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# DESCRIPTION
-# This module provides the OpenEmbedded partition object definitions.
-#
-# AUTHORS
-# Tom Zanussi <tom.zanussi (at] linux.intel.com>
-#
-
-import shutil
-import os
-import tempfile
-
-from pykickstart.commands.partition import *
-from wic.utils.oe.misc import *
-from wic.kickstart.custom_commands import *
-from wic.plugin import pluginmgr
-
-partition_methods = {
- "do_stage_partition":None,
- "do_prepare_partition":None,
- "do_configure_partition":None,
-}
-
-class Wic_PartData(Mic_PartData):
- removedKeywords = Mic_PartData.removedKeywords
- removedAttrs = Mic_PartData.removedAttrs
-
- def __init__(self, *args, **kwargs):
- Mic_PartData.__init__(self, *args, **kwargs)
- self.deleteRemovedAttrs()
- self.source = kwargs.get("source", None)
- self.sourceparams = kwargs.get("sourceparams", None)
- self.rootfs = kwargs.get("rootfs-dir", None)
- self.no_table = kwargs.get("no-table", False)
- self.extra_space = kwargs.get("extra-space", "10M")
- self.overhead_factor = kwargs.get("overhead-factor", 1.3)
- self.source_file = ""
- self.size = 0
-
- def _getArgsAsStr(self):
- retval = Mic_PartData._getArgsAsStr(self)
-
- if self.source:
- retval += " --source=%s" % self.source
- if self.sourceparams:
- retval += " --sourceparams=%s" % self.sourceparams
- if self.rootfs:
- retval += " --rootfs-dir=%s" % self.rootfs
- if self.no_table:
- retval += " --no-table"
- retval += " --extra-space=%d" % self.extra_space
- retval += " --overhead-factor=%f" % self.overhead_factor
-
- return retval
-
- def get_rootfs(self):
- """
- Acessor for rootfs dir
- """
- return self.rootfs
-
- def set_rootfs(self, rootfs):
- """
- Acessor for actual rootfs dir, which must be set by source
- plugins.
- """
- self.rootfs = rootfs
-
- def get_size(self):
- """
- Accessor for partition size, 0 or --size before set_size().
- """
- return self.size
-
- def set_size(self, size):
- """
- Accessor for actual partition size, which must be set by source
- plugins.
- """
- self.size = size
-
- def set_source_file(self, source_file):
- """
- Accessor for source_file, the location of the generated partition
- image, which must be set by source plugins.
- """
- self.source_file = source_file
-
- def get_extra_block_count(self, current_blocks):
- """
- The --size param is reflected in self.size (in kB), and we already
- have current_blocks (1k) blocks, calculate and return the
- number of (1k) blocks we need to add to get to --size, 0 if
- we're already there or beyond.
- """
- msger.debug("Requested partition size for %s: %d" % \
- (self.mountpoint, self.size))
-
- if not self.size:
- return 0
-
- requested_blocks = self.size
-
- msger.debug("Requested blocks %d, current_blocks %d" % \
- (requested_blocks, current_blocks))
-
- if requested_blocks > current_blocks:
- return requested_blocks - current_blocks
- else:
- return 0
-
- def prepare(self, cr, cr_workdir, oe_builddir, rootfs_dir, bootimg_dir,
- kernel_dir, native_sysroot):
- """
- Prepare content for individual partitions, depending on
- partition command parameters.
- """
- self.sourceparams_dict = {}
-
- if self.sourceparams:
- self.sourceparams_dict = parse_sourceparams(self.sourceparams)
-
- if not self.source:
- if not self.size:
- msger.error("The %s partition has a size of zero. Please specify a non-zero --size for that partition." % self.mountpoint)
- if self.fstype and self.fstype == "swap":
- self.prepare_swap_partition(cr_workdir, oe_builddir,
- native_sysroot)
- elif self.fstype:
- self.prepare_empty_partition(cr_workdir, oe_builddir,
- native_sysroot)
- return
-
- plugins = pluginmgr.get_source_plugins()
-
- if self.source not in plugins:
- msger.error("The '%s' --source specified for %s doesn't exist.\n\tSee 'wic list source-plugins' for a list of available --sources.\n\tSee 'wic help source-plugins' for details on adding a new source plugin." % (self.source, self.mountpoint))
-
- self._source_methods = pluginmgr.get_source_plugin_methods(self.source, partition_methods)
- self._source_methods["do_configure_partition"](self, self.sourceparams_dict,
- cr, cr_workdir,
- oe_builddir,
- bootimg_dir,
- kernel_dir,
- native_sysroot)
- self._source_methods["do_stage_partition"](self, self.sourceparams_dict,
- cr, cr_workdir,
- oe_builddir,
- bootimg_dir, kernel_dir,
- native_sysroot)
- self._source_methods["do_prepare_partition"](self, self.sourceparams_dict,
- cr, cr_workdir,
- oe_builddir,
- bootimg_dir, kernel_dir, rootfs_dir,
- native_sysroot)
-
- def prepare_rootfs_from_fs_image(self, cr_workdir, oe_builddir,
- rootfs_dir):
- """
- Handle an already-created partition e.g. xxx.ext3
- """
- rootfs = oe_builddir
- du_cmd = "du -Lbks %s" % rootfs
- out = exec_cmd(du_cmd)
- rootfs_size = out.split()[0]
-
- self.size = rootfs_size
- self.source_file = rootfs
-
- def prepare_rootfs(self, cr_workdir, oe_builddir, rootfs_dir,
- native_sysroot):
- """
- Prepare content for a rootfs partition i.e. create a partition
- and fill it from a /rootfs dir.
-
- Currently handles ext2/3/4, btrfs and vfat.
- """
- p_prefix = os.environ.get("PSEUDO_PREFIX", "%s/usr" % native_sysroot)
- p_localstatedir = os.environ.get("PSEUDO_LOCALSTATEDIR",
- "%s/../pseudo" % rootfs_dir)
- p_passwd = os.environ.get("PSEUDO_PASSWD", rootfs_dir)
- p_nosymlinkexp = os.environ.get("PSEUDO_NOSYMLINKEXP", "1")
- pseudo = "export PSEUDO_PREFIX=%s;" % p_prefix
- pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % p_localstatedir
- pseudo += "export PSEUDO_PASSWD=%s;" % p_passwd
- pseudo += "export PSEUDO_NOSYMLINKEXP=%s;" % p_nosymlinkexp
- pseudo += "%s/usr/bin/pseudo " % native_sysroot
-
- if self.fstype.startswith("ext"):
- return self.prepare_rootfs_ext(cr_workdir, oe_builddir,
- rootfs_dir, native_sysroot,
- pseudo)
- elif self.fstype.startswith("btrfs"):
- return self.prepare_rootfs_btrfs(cr_workdir, oe_builddir,
- rootfs_dir, native_sysroot,
- pseudo)
-
- elif self.fstype.startswith("vfat"):
- return self.prepare_rootfs_vfat(cr_workdir, oe_builddir,
- rootfs_dir, native_sysroot,
- pseudo)
- elif self.fstype.startswith("squashfs"):
- return self.prepare_rootfs_squashfs(cr_workdir, oe_builddir,
- rootfs_dir, native_sysroot,
- pseudo)
-
- def prepare_rootfs_ext(self, cr_workdir, oe_builddir, rootfs_dir,
- native_sysroot, pseudo):
- """
- Prepare content for an ext2/3/4 rootfs partition.
- """
-
- image_rootfs = rootfs_dir
- rootfs = "%s/rootfs_%s.%s" % (cr_workdir, self.label ,self.fstype)
-
- os.path.isfile(rootfs) and os.remove(rootfs)
- du_cmd = "du -ks %s" % image_rootfs
- out = exec_cmd(du_cmd)
- actual_rootfs_size = int(out.split()[0])
-
- extra_blocks = self.get_extra_block_count(actual_rootfs_size)
- if extra_blocks < self.extra_space:
- extra_blocks = self.extra_space
-
- rootfs_size = actual_rootfs_size + extra_blocks
- rootfs_size *= self.overhead_factor
-
- msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \
- (extra_blocks, self.mountpoint, rootfs_size))
-
- dd_cmd = "dd if=/dev/zero of=%s bs=1024 seek=%d count=0 bs=1k" % \
- (rootfs, rootfs_size)
- exec_cmd(dd_cmd)
-
- extra_imagecmd = "-i 8192"
-
- label_str = ""
- if (self.label):
- label_str = "-L %s" % self.label
-
- mkfs_cmd = "mkfs.%s -F %s %s %s -d %s" % \
- (self.fstype, extra_imagecmd, rootfs, label_str, image_rootfs)
- (rc, out) = exec_native_cmd(pseudo + mkfs_cmd, native_sysroot)
- if rc:
- print "rootfs_dir: %s" % rootfs_dir
- msger.error("ERROR: mkfs.%s returned '%s' instead of 0 (which you probably don't want to ignore, use --debug for details) when creating filesystem from rootfs directory: %s" % (self.fstype, rc, rootfs_dir))
-
- # get the rootfs size in the right units for kickstart (kB)
- du_cmd = "du -Lbks %s" % rootfs
- out = exec_cmd(du_cmd)
- rootfs_size = out.split()[0]
-
- self.size = rootfs_size
- self.source_file = rootfs
-
- return 0
-
- def prepare_rootfs_btrfs(self, cr_workdir, oe_builddir, rootfs_dir,
- native_sysroot, pseudo):
- """
- Prepare content for a btrfs rootfs partition.
-
- Currently handles ext2/3/4 and btrfs.
- """
- image_rootfs = rootfs_dir
- rootfs = "%s/rootfs_%s.%s" % (cr_workdir, self.label, self.fstype)
-
- os.path.isfile(rootfs) and os.remove(rootfs)
- du_cmd = "du -ks %s" % image_rootfs
- out = exec_cmd(du_cmd)
- actual_rootfs_size = int(out.split()[0])
-
- extra_blocks = self.get_extra_block_count(actual_rootfs_size)
- if extra_blocks < self.extra_space:
- extra_blocks = self.extra_space
-
- rootfs_size = actual_rootfs_size + extra_blocks
- rootfs_size *= self.overhead_factor
-
- msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \
- (extra_blocks, self.mountpoint, rootfs_size))
-
- dd_cmd = "dd if=/dev/zero of=%s bs=1024 seek=%d count=0 bs=1k" % \
- (rootfs, rootfs_size)
- exec_cmd(dd_cmd)
-
- label_str = ""
- if (self.label):
- label_str = "-L %s" % self.label
-
- mkfs_cmd = "mkfs.%s -b %d -r %s %s %s" % \
- (self.fstype, rootfs_size * 1024, image_rootfs, label_str, rootfs)
- (rc, out) = exec_native_cmd(pseudo + mkfs_cmd, native_sysroot)
- if rc:
- msger.error("ERROR: mkfs.%s returned '%s' instead of 0 (which you probably don't want to ignore, use --debug for details) when creating filesystem from rootfs directory: %s" % (self.fstype, rc, rootfs_dir))
-
- # get the rootfs size in the right units for kickstart (kB)
- du_cmd = "du -Lbks %s" % rootfs
- out = exec_cmd(du_cmd)
- rootfs_size = out.split()[0]
-
- self.size = rootfs_size
- self.source_file = rootfs
-
- def prepare_rootfs_vfat(self, cr_workdir, oe_builddir, rootfs_dir,
- native_sysroot, pseudo):
- """
- Prepare content for a vfat rootfs partition.
- """
- image_rootfs = rootfs_dir
- rootfs = "%s/rootfs_%s.%s" % (cr_workdir, self.label, self.fstype)
-
- os.path.isfile(rootfs) and os.remove(rootfs)
- du_cmd = "du -bks %s" % image_rootfs
- out = exec_cmd(du_cmd)
- blocks = int(out.split()[0])
-
- extra_blocks = self.get_extra_block_count(blocks)
- if extra_blocks < self.extra_space:
- extra_blocks = self.extra_space
-
- blocks += extra_blocks
-
- msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \
- (extra_blocks, self.mountpoint, blocks))
-
- # Ensure total sectors is an integral number of sectors per
- # track or mcopy will complain. Sectors are 512 bytes, and we
- # generate images with 32 sectors per track. This calculation
- # is done in blocks, thus the mod by 16 instead of 32. Apply
- # sector count fix only when needed.
- if blocks % 16 != 0:
- blocks += (16 - (blocks % 16))
-
- label_str = "-n boot"
- if (self.label):
- label_str = "-n %s" % self.label
-
- dosfs_cmd = "mkdosfs %s -S 512 -C %s %d" % (label_str, rootfs, blocks)
- exec_native_cmd(dosfs_cmd, native_sysroot)
-
- mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, image_rootfs)
- rc, out = exec_native_cmd(mcopy_cmd, native_sysroot)
- if rc:
- msger.error("ERROR: mcopy returned '%s' instead of 0 (which you probably don't want to ignore, use --debug for details)" % rc)
-
- chmod_cmd = "chmod 644 %s" % rootfs
- exec_cmd(chmod_cmd)
-
- # get the rootfs size in the right units for kickstart (kB)
- du_cmd = "du -Lbks %s" % rootfs
- out = exec_cmd(du_cmd)
- rootfs_size = out.split()[0]
-
- self.set_size(rootfs_size)
- self.set_source_file(rootfs)
-
- def prepare_rootfs_squashfs(self, cr_workdir, oe_builddir, rootfs_dir,
- native_sysroot, pseudo):
- """
- Prepare content for a squashfs rootfs partition.
- """
- image_rootfs = rootfs_dir
- rootfs = "%s/rootfs_%s.%s" % (cr_workdir, self.label ,self.fstype)
-
- os.path.isfile(rootfs) and os.remove(rootfs)
- squashfs_cmd = "mksquashfs %s %s -noappend" % \
- (image_rootfs, rootfs)
- exec_native_cmd(pseudo + squashfs_cmd, native_sysroot)
-
- # get the rootfs size in the right units for kickstart (kB)
- du_cmd = "du -Lbks %s" % rootfs
- out = exec_cmd(du_cmd)
- rootfs_size = out.split()[0]
-
- self.size = rootfs_size
- self.source_file = rootfs
-
- return 0
-
- def prepare_empty_partition(self, cr_workdir, oe_builddir, native_sysroot):
- """
- Prepare an empty partition.
- """
- if self.fstype.startswith("ext"):
- return self.prepare_empty_partition_ext(cr_workdir, oe_builddir,
- native_sysroot)
- elif self.fstype.startswith("btrfs"):
- return self.prepare_empty_partition_btrfs(cr_workdir, oe_builddir,
- native_sysroot)
- elif self.fstype.startswith("vfat"):
- return self.prepare_empty_partition_vfat(cr_workdir, oe_builddir,
- native_sysroot)
- elif self.fstype.startswith("squashfs"):
- return self.prepare_empty_partition_squashfs(cr_workdir, oe_builddir,
- native_sysroot)
-
- def prepare_empty_partition_ext(self, cr_workdir, oe_builddir,
- native_sysroot):
- """
- Prepare an empty ext2/3/4 partition.
- """
- fs = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype)
-
- os.path.isfile(fs) and os.remove(fs)
- dd_cmd = "dd if=/dev/zero of=%s bs=1k seek=%d count=0" % \
- (fs, self.size)
- exec_cmd(dd_cmd)
-
- extra_imagecmd = "-i 8192"
-
- label_str = ""
- if (self.label):
- label_str = "-L %s" % self.label
-
- mkfs_cmd = "mkfs.%s -F %s %s %s" % \
- (self.fstype, extra_imagecmd, label_str, fs)
- (rc, out) = exec_native_cmd(mkfs_cmd, native_sysroot)
- if rc:
- msger.error("ERROR: mkfs.%s returned '%s' instead of 0 (which you probably don't want to ignore, use --debug for details)" % (self.fstype, rc))
-
- self.source_file = fs
-
- return 0
-
- def prepare_empty_partition_btrfs(self, cr_workdir, oe_builddir,
- native_sysroot):
- """
- Prepare an empty btrfs partition.
- """
- fs = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype)
-
- os.path.isfile(fs) and os.remove(fs)
- dd_cmd = "dd if=/dev/zero of=%s bs=1k seek=%d count=0" % \
- (fs, self.size)
- exec_cmd(dd_cmd)
-
- label_str = ""
- if (self.label):
- label_str = "-L %s" % self.label
-
- mkfs_cmd = "mkfs.%s -b %d %s %s" % \
- (self.fstype, self.size * 1024, label_str, fs)
- (rc, out) = exec_native_cmd(mkfs_cmd, native_sysroot)
- if rc:
- msger.error("ERROR: mkfs.%s returned '%s' instead of 0 (which you probably don't want to ignore, use --debug for details)" % (self.fstype, rc))
-
- self.source_file = fs
-
- return 0
-
- def prepare_empty_partition_vfat(self, cr_workdir, oe_builddir,
- native_sysroot):
- """
- Prepare an empty vfat partition.
- """
- fs = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype)
- os.path.isfile(fs) and os.remove(fs)
-
- blocks = self.size
-
- label_str = "-n boot"
- if (self.label):
- label_str = "-n %s" % self.label
-
- dosfs_cmd = "mkdosfs %s -S 512 -C %s %d" % (label_str, fs, blocks)
- exec_native_cmd(dosfs_cmd, native_sysroot)
-
- chmod_cmd = "chmod 644 %s" % fs
- exec_cmd(chmod_cmd)
-
- self.source_file = fs
-
- return 0
-
- def prepare_empty_partition_squashfs(self, cr_workdir, oe_builddir,
- native_sysroot):
- """
- Prepare an empty squashfs partition.
- """
- msger.warning("Creating of an empty squashfs %s partition was attempted. " \
- "Proceeding as requested." % self.mountpoint)
-
- fs = "%s/fs_%s.%s" % (cr_workdir, self.label, self.fstype)
- os.path.isfile(fs) and os.remove(fs)
-
- # it is not possible to create a squashfs without source data,
- # thus prepare an empty temp dir that is used as source
- tmpdir = tempfile.mkdtemp()
-
- squashfs_cmd = "mksquashfs %s %s -noappend" % \
- (tmpdir, fs)
- exec_native_cmd(squashfs_cmd, native_sysroot)
-
- os.rmdir(tmpdir)
-
- # get the rootfs size in the right units for kickstart (kB)
- du_cmd = "du -Lbks %s" % fs
- out = exec_cmd(du_cmd)
- fs_size = out.split()[0]
-
- self.size = fs_size
- self.source_file = fs
-
- return 0
-
- def prepare_swap_partition(self, cr_workdir, oe_builddir, native_sysroot):
- """
- Prepare a swap partition.
- """
- fs = "%s/fs.%s" % (cr_workdir, self.fstype)
-
- dd_cmd = "dd if=/dev/zero of=%s bs=1k seek=%d count=0" % \
- (fs, self.size)
- exec_cmd(dd_cmd)
-
- import uuid
- label_str = ""
- if self.label:
- label_str = "-L %s" % self.label
- mkswap_cmd = "mkswap %s -U %s %s" % (label_str, str(uuid.uuid1()), fs)
- exec_native_cmd(mkswap_cmd, native_sysroot)
-
- self.source_file = fs
-
- return 0
-
-class Wic_Partition(Mic_Partition):
- removedKeywords = Mic_Partition.removedKeywords
- removedAttrs = Mic_Partition.removedAttrs
-
- def _getParser(self):
- def overhead_cb (option, opt_str, value, parser):
- if (value < 1):
- raise OptionValueError("Option %s: invalid value: %r" % (option, value))
- setattr(parser.values, option.dest, value)
-
- op = Mic_Partition._getParser(self)
- # use specified source file to fill the partition
- # and calculate partition size
- op.add_option("--source", type="string", action="store",
- dest="source", default=None)
- # comma-separated list of param=value pairs
- op.add_option("--sourceparams", type="string", action="store",
- dest="sourceparams", default=None)
- # use specified rootfs path to fill the partition
- op.add_option("--rootfs-dir", type="string", action="store",
- dest="rootfs", default=None)
- # wether to add the partition in the partition table
- op.add_option("--no-table", dest="no_table", action="store_true",
- default=False)
- # extra space beyond the partition size
- op.add_option("--extra-space", dest="extra_space", action="store",
- type="size", nargs=1, default="10M")
- op.add_option("--overhead-factor", dest="overhead_factor",
- action="callback", callback=overhead_cb, type="float",
- nargs=1, default=1.3)
- return op
diff --git a/scripts/lib/wic/kickstart/custom_commands/wicboot.py b/scripts/lib/wic/kickstart/custom_commands/wicboot.py
deleted file mode 100644
index f1914169d8..0000000000
--- a/scripts/lib/wic/kickstart/custom_commands/wicboot.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (c) 2014, Intel Corporation.
-# All rights reserved.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# DESCRIPTION
-# This module provides the OpenEmbedded bootloader object definitions.
-#
-# AUTHORS
-# Tom Zanussi <tom.zanussi (at] linux.intel.com>
-#
-
-from pykickstart.base import *
-from pykickstart.errors import *
-from pykickstart.options import *
-from pykickstart.commands.bootloader import *
-
-from wic.kickstart.custom_commands.micboot import *
-
-class Wic_Bootloader(Mic_Bootloader):
- def __init__(self, writePriority=10, appendLine="", driveorder=None,
- forceLBA=False, location="", md5pass="", password="",
- upgrade=False, menus=""):
- Mic_Bootloader.__init__(self, writePriority, appendLine, driveorder,
- forceLBA, location, md5pass, password, upgrade)
-
- self.source = ""
-
- def _getArgsAsStr(self):
- retval = Mic_Bootloader._getArgsAsStr(self)
-
- if self.source:
- retval += " --source=%s" % self.source
-
- return retval
-
- def _getParser(self):
- op = Mic_Bootloader._getParser(self)
- # use specified source plugin to implement bootloader-specific methods
- op.add_option("--source", type="string", action="store",
- dest="source", default=None)
- return op
-
diff --git a/scripts/lib/wic/ksparser.py b/scripts/lib/wic/ksparser.py
new file mode 100644
index 0000000000..d026caad0f
--- /dev/null
+++ b/scripts/lib/wic/ksparser.py
@@ -0,0 +1,227 @@
+#!/usr/bin/env python -tt
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (c) 2016 Intel, Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation; version 2 of the License
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc., 59
+# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+# DESCRIPTION
+# This module provides parser for kickstart format
+#
+# AUTHORS
+# Ed Bartosh <ed.bartosh> (at] linux.intel.com>
+
+"""Kickstart parser module."""
+
+import os
+import shlex
+import logging
+
+from argparse import ArgumentParser, ArgumentError, ArgumentTypeError
+
+from wic.engine import find_canned
+from wic.partition import Partition
+
+logger = logging.getLogger('wic')
+
+class KickStartError(Exception):
+ """Custom exception."""
+ pass
+
+class KickStartParser(ArgumentParser):
+ """
+ This class overwrites error method to throw exception
+ instead of producing usage message(default argparse behavior).
+ """
+ def error(self, message):
+ raise ArgumentError(None, message)
+
+def sizetype(arg):
+ """
+ Custom type for ArgumentParser
+ Converts size string in <num>[K|k|M|G] format into the integer value
+ """
+ if arg.isdigit():
+ return int(arg) * 1024
+
+ if not arg[:-1].isdigit():
+ raise ArgumentTypeError("Invalid size: %r" % arg)
+
+ size = int(arg[:-1])
+ if arg.endswith("k") or arg.endswith("K"):
+ return size
+ if arg.endswith("M"):
+ return size * 1024
+ if arg.endswith("G"):
+ return size * 1024 * 1024
+
+ raise ArgumentTypeError("Invalid size: %r" % arg)
+
+def overheadtype(arg):
+ """
+ Custom type for ArgumentParser
+ Converts overhead string to float and checks if it's bigger than 1.0
+ """
+ try:
+ result = float(arg)
+ except ValueError:
+ raise ArgumentTypeError("Invalid value: %r" % arg)
+
+ if result < 1.0:
+ raise ArgumentTypeError("Overhead factor should be > 1.0" % arg)
+
+ return result
+
+def cannedpathtype(arg):
+ """
+ Custom type for ArgumentParser
+ Tries to find file in the list of canned wks paths
+ """
+ scripts_path = os.path.abspath(os.path.dirname(__file__) + '../../..')
+ result = find_canned(scripts_path, arg)
+ if not result:
+ raise ArgumentTypeError("file not found: %s" % arg)
+ return result
+
+def systemidtype(arg):
+ """
+ Custom type for ArgumentParser
+ Checks if the argument sutisfies system id requirements,
+ i.e. if it's one byte long integer > 0
+ """
+ error = "Invalid system type: %s. must be hex "\
+ "between 0x1 and 0xFF" % arg
+ try:
+ result = int(arg, 16)
+ except ValueError:
+ raise ArgumentTypeError(error)
+
+ if result <= 0 or result > 0xff:
+ raise ArgumentTypeError(error)
+
+ return arg
+
+class KickStart():
+ """"Kickstart parser implementation."""
+
+ DEFAULT_EXTRA_SPACE = 10*1024
+ DEFAULT_OVERHEAD_FACTOR = 1.3
+
+ def __init__(self, confpath):
+
+ self.partitions = []
+ self.bootloader = None
+ self.lineno = 0
+ self.partnum = 0
+
+ parser = KickStartParser()
+ subparsers = parser.add_subparsers()
+
+ part = subparsers.add_parser('part')
+ part.add_argument('mountpoint', nargs='?')
+ part.add_argument('--active', action='store_true')
+ part.add_argument('--align', type=int)
+ part.add_argument('--exclude-path', nargs='+')
+ part.add_argument("--extra-space", type=sizetype)
+ part.add_argument('--fsoptions', dest='fsopts')
+ part.add_argument('--fstype', default='vfat',
+ choices=('ext2', 'ext3', 'ext4', 'btrfs',
+ 'squashfs', 'vfat', 'msdos', 'swap'))
+ part.add_argument('--label')
+ part.add_argument('--no-table', action='store_true')
+ part.add_argument('--ondisk', '--ondrive', dest='disk', default='sda')
+ part.add_argument("--overhead-factor", type=overheadtype)
+ part.add_argument('--part-type')
+ part.add_argument('--rootfs-dir')
+
+ # --size and --fixed-size cannot be specified together; options
+ # ----extra-space and --overhead-factor should also raise a parser
+ # --error, but since nesting mutually exclusive groups does not work,
+ # ----extra-space/--overhead-factor are handled later
+ sizeexcl = part.add_mutually_exclusive_group()
+ sizeexcl.add_argument('--size', type=sizetype, default=0)
+ sizeexcl.add_argument('--fixed-size', type=sizetype, default=0)
+
+ part.add_argument('--source')
+ part.add_argument('--sourceparams')
+ part.add_argument('--system-id', type=systemidtype)
+ part.add_argument('--use-uuid', action='store_true')
+ part.add_argument('--uuid')
+
+ bootloader = subparsers.add_parser('bootloader')
+ bootloader.add_argument('--append')
+ bootloader.add_argument('--configfile')
+ bootloader.add_argument('--ptable', choices=('msdos', 'gpt'),
+ default='msdos')
+ bootloader.add_argument('--timeout', type=int)
+ bootloader.add_argument('--source')
+
+ include = subparsers.add_parser('include')
+ include.add_argument('path', type=cannedpathtype)
+
+ self._parse(parser, confpath)
+ if not self.bootloader:
+ logger.warning('bootloader config not specified, using defaults\n')
+ self.bootloader = bootloader.parse_args([])
+
+ def _parse(self, parser, confpath):
+ """
+ Parse file in .wks format using provided parser.
+ """
+ with open(confpath) as conf:
+ lineno = 0
+ for line in conf:
+ line = line.strip()
+ lineno += 1
+ if line and line[0] != '#':
+ try:
+ line_args = shlex.split(line)
+ parsed = parser.parse_args(line_args)
+ except ArgumentError as err:
+ raise KickStartError('%s:%d: %s' % \
+ (confpath, lineno, err))
+ if line.startswith('part'):
+ # using ArgumentParser one cannot easily tell if option
+ # was passed as argument, if said option has a default
+ # value; --overhead-factor/--extra-space cannot be used
+ # with --fixed-size, so at least detect when these were
+ # passed with non-0 values ...
+ if parsed.fixed_size:
+ if parsed.overhead_factor or parsed.extra_space:
+ err = "%s:%d: arguments --overhead-factor and --extra-space not "\
+ "allowed with argument --fixed-size" \
+ % (confpath, lineno)
+ raise KickStartError(err)
+ else:
+ # ... and provide defaults if not using
+ # --fixed-size iff given option was not used
+ # (again, one cannot tell if option was passed but
+ # with value equal to 0)
+ if '--overhead-factor' not in line_args:
+ parsed.overhead_factor = self.DEFAULT_OVERHEAD_FACTOR
+ if '--extra-space' not in line_args:
+ parsed.extra_space = self.DEFAULT_EXTRA_SPACE
+
+ self.partnum += 1
+ self.partitions.append(Partition(parsed, self.partnum))
+ elif line.startswith('include'):
+ self._parse(parser, parsed.path)
+ elif line.startswith('bootloader'):
+ if not self.bootloader:
+ self.bootloader = parsed
+ else:
+ err = "%s:%d: more than one bootloader specified" \
+ % (confpath, lineno)
+ raise KickStartError(err)
diff --git a/scripts/lib/wic/msger.py b/scripts/lib/wic/msger.py
deleted file mode 100644
index 9f557e7b9a..0000000000
--- a/scripts/lib/wic/msger.py
+++ /dev/null
@@ -1,309 +0,0 @@
-#!/usr/bin/env python -tt
-# vim: ai ts=4 sts=4 et sw=4
-#
-# Copyright (c) 2009, 2010, 2011 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-import os,sys
-import re
-import time
-
-__ALL__ = ['set_mode',
- 'get_loglevel',
- 'set_loglevel',
- 'set_logfile',
- 'raw',
- 'debug',
- 'verbose',
- 'info',
- 'warning',
- 'error',
- 'ask',
- 'pause',
- ]
-
-# COLORs in ANSI
-INFO_COLOR = 32 # green
-WARN_COLOR = 33 # yellow
-ERR_COLOR = 31 # red
-ASK_COLOR = 34 # blue
-NO_COLOR = 0
-
-PREFIX_RE = re.compile('^<(.*?)>\s*(.*)', re.S)
-
-INTERACTIVE = True
-
-LOG_LEVEL = 1
-LOG_LEVELS = {
- 'quiet': 0,
- 'normal': 1,
- 'verbose': 2,
- 'debug': 3,
- 'never': 4,
- }
-
-LOG_FILE_FP = None
-LOG_CONTENT = ''
-CATCHERR_BUFFILE_FD = -1
-CATCHERR_BUFFILE_PATH = None
-CATCHERR_SAVED_2 = -1
-
-def _general_print(head, color, msg = None, stream = None, level = 'normal'):
- global LOG_CONTENT
- if not stream:
- stream = sys.stdout
-
- if LOG_LEVELS[level] > LOG_LEVEL:
- # skip
- return
-
- # encode raw 'unicode' str to utf8 encoded str
- if msg and isinstance(msg, unicode):
- msg = msg.encode('utf-8', 'ignore')
-
- errormsg = ''
- if CATCHERR_BUFFILE_FD > 0:
- size = os.lseek(CATCHERR_BUFFILE_FD , 0, os.SEEK_END)
- os.lseek(CATCHERR_BUFFILE_FD, 0, os.SEEK_SET)
- errormsg = os.read(CATCHERR_BUFFILE_FD, size)
- os.ftruncate(CATCHERR_BUFFILE_FD, 0)
-
- # append error msg to LOG
- if errormsg:
- LOG_CONTENT += errormsg
-
- # append normal msg to LOG
- save_msg = msg.strip() if msg else None
- if save_msg:
- timestr = time.strftime("[%m/%d %H:%M:%S %Z] ", time.localtime())
- LOG_CONTENT += timestr + save_msg + '\n'
-
- if errormsg:
- _color_print('', NO_COLOR, errormsg, stream, level)
-
- _color_print(head, color, msg, stream, level)
-
-def _color_print(head, color, msg, stream, level):
- colored = True
- if color == NO_COLOR or \
- not stream.isatty() or \
- os.getenv('ANSI_COLORS_DISABLED') is not None:
- colored = False
-
- if head.startswith('\r'):
- # need not \n at last
- newline = False
- else:
- newline = True
-
- if colored:
- head = '\033[%dm%s:\033[0m ' %(color, head)
- if not newline:
- # ESC cmd to clear line
- head = '\033[2K' + head
- else:
- if head:
- head += ': '
- if head.startswith('\r'):
- head = head.lstrip()
- newline = True
-
- if msg is not None:
- if isinstance(msg, unicode):
- msg = msg.encode('utf8', 'ignore')
-
- stream.write('%s%s' % (head, msg))
- if newline:
- stream.write('\n')
-
- stream.flush()
-
-def _color_perror(head, color, msg, level = 'normal'):
- if CATCHERR_BUFFILE_FD > 0:
- _general_print(head, color, msg, sys.stdout, level)
- else:
- _general_print(head, color, msg, sys.stderr, level)
-
-def _split_msg(head, msg):
- if isinstance(msg, list):
- msg = '\n'.join(map(str, msg))
-
- if msg.startswith('\n'):
- # means print \n at first
- msg = msg.lstrip()
- head = '\n' + head
-
- elif msg.startswith('\r'):
- # means print \r at first
- msg = msg.lstrip()
- head = '\r' + head
-
- m = PREFIX_RE.match(msg)
- if m:
- head += ' <%s>' % m.group(1)
- msg = m.group(2)
-
- return head, msg
-
-def get_loglevel():
- return (k for k,v in LOG_LEVELS.items() if v==LOG_LEVEL).next()
-
-def set_loglevel(level):
- global LOG_LEVEL
- if level not in LOG_LEVELS:
- # no effect
- return
-
- LOG_LEVEL = LOG_LEVELS[level]
-
-def set_interactive(mode=True):
- global INTERACTIVE
- if mode:
- INTERACTIVE = True
- else:
- INTERACTIVE = False
-
-def log(msg=''):
- # log msg to LOG_CONTENT then save to logfile
- global LOG_CONTENT
- if msg:
- LOG_CONTENT += msg
-
-def raw(msg=''):
- _general_print('', NO_COLOR, msg)
-
-def info(msg):
- head, msg = _split_msg('Info', msg)
- _general_print(head, INFO_COLOR, msg)
-
-def verbose(msg):
- head, msg = _split_msg('Verbose', msg)
- _general_print(head, INFO_COLOR, msg, level = 'verbose')
-
-def warning(msg):
- head, msg = _split_msg('Warning', msg)
- _color_perror(head, WARN_COLOR, msg)
-
-def debug(msg):
- head, msg = _split_msg('Debug', msg)
- _color_perror(head, ERR_COLOR, msg, level = 'debug')
-
-def error(msg):
- head, msg = _split_msg('Error', msg)
- _color_perror(head, ERR_COLOR, msg)
- sys.exit(1)
-
-def ask(msg, default=True):
- _general_print('\rQ', ASK_COLOR, '')
- try:
- if default:
- msg += '(Y/n) '
- else:
- msg += '(y/N) '
- if INTERACTIVE:
- while True:
- repl = raw_input(msg)
- if repl.lower() == 'y':
- return True
- elif repl.lower() == 'n':
- return False
- elif not repl.strip():
- # <Enter>
- return default
-
- # else loop
- else:
- if default:
- msg += ' Y'
- else:
- msg += ' N'
- _general_print('', NO_COLOR, msg)
-
- return default
- except KeyboardInterrupt:
- sys.stdout.write('\n')
- sys.exit(2)
-
-def choice(msg, choices, default=0):
- if default >= len(choices):
- return None
- _general_print('\rQ', ASK_COLOR, '')
- try:
- msg += " [%s] " % '/'.join(choices)
- if INTERACTIVE:
- while True:
- repl = raw_input(msg)
- if repl in choices:
- return repl
- elif not repl.strip():
- return choices[default]
- else:
- msg += choices[default]
- _general_print('', NO_COLOR, msg)
-
- return choices[default]
- except KeyboardInterrupt:
- sys.stdout.write('\n')
- sys.exit(2)
-
-def pause(msg=None):
- if INTERACTIVE:
- _general_print('\rQ', ASK_COLOR, '')
- if msg is None:
- msg = 'press <ENTER> to continue ...'
- raw_input(msg)
-
-def set_logfile(fpath):
- global LOG_FILE_FP
-
- def _savelogf():
- if LOG_FILE_FP:
- fp = open(LOG_FILE_FP, 'w')
- fp.write(LOG_CONTENT)
- fp.close()
-
- if LOG_FILE_FP is not None:
- warning('duplicate log file configuration')
-
- LOG_FILE_FP = fpath
-
- import atexit
- atexit.register(_savelogf)
-
-def enable_logstderr(fpath):
- global CATCHERR_BUFFILE_FD
- global CATCHERR_BUFFILE_PATH
- global CATCHERR_SAVED_2
-
- if os.path.exists(fpath):
- os.remove(fpath)
- CATCHERR_BUFFILE_PATH = fpath
- CATCHERR_BUFFILE_FD = os.open(CATCHERR_BUFFILE_PATH, os.O_RDWR|os.O_CREAT)
- CATCHERR_SAVED_2 = os.dup(2)
- os.dup2(CATCHERR_BUFFILE_FD, 2)
-
-def disable_logstderr():
- global CATCHERR_BUFFILE_FD
- global CATCHERR_BUFFILE_PATH
- global CATCHERR_SAVED_2
-
- raw(msg = None) # flush message buffer and print it.
- os.dup2(CATCHERR_SAVED_2, 2)
- os.close(CATCHERR_SAVED_2)
- os.close(CATCHERR_BUFFILE_FD)
- os.unlink(CATCHERR_BUFFILE_PATH)
- CATCHERR_BUFFILE_FD = -1
- CATCHERR_BUFFILE_PATH = None
- CATCHERR_SAVED_2 = -1
diff --git a/scripts/lib/wic/partition.py b/scripts/lib/wic/partition.py
new file mode 100644
index 0000000000..939e66731b
--- /dev/null
+++ b/scripts/lib/wic/partition.py
@@ -0,0 +1,411 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (c) 2013-2016 Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# This module provides the OpenEmbedded partition object definitions.
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+# Ed Bartosh <ed.bartosh> (at] linux.intel.com>
+
+import logging
+import os
+import tempfile
+
+from wic import WicError
+from wic.utils.misc import exec_cmd, exec_native_cmd, get_bitbake_var
+from wic.pluginbase import PluginMgr
+
+logger = logging.getLogger('wic')
+
+class Partition():
+
+ def __init__(self, args, lineno):
+ self.args = args
+ self.active = args.active
+ self.align = args.align
+ self.disk = args.disk
+ self.device = None
+ self.extra_space = args.extra_space
+ self.exclude_path = args.exclude_path
+ self.fsopts = args.fsopts
+ self.fstype = args.fstype
+ self.label = args.label
+ self.mountpoint = args.mountpoint
+ self.no_table = args.no_table
+ self.num = None
+ self.overhead_factor = args.overhead_factor
+ self.part_type = args.part_type
+ self.rootfs_dir = args.rootfs_dir
+ self.size = args.size
+ self.fixed_size = args.fixed_size
+ self.source = args.source
+ self.sourceparams = args.sourceparams
+ self.system_id = args.system_id
+ self.use_uuid = args.use_uuid
+ self.uuid = args.uuid
+
+ self.lineno = lineno
+ self.source_file = ""
+ self.sourceparams_dict = {}
+
+ def get_extra_block_count(self, current_blocks):
+ """
+ The --size param is reflected in self.size (in kB), and we already
+ have current_blocks (1k) blocks, calculate and return the
+ number of (1k) blocks we need to add to get to --size, 0 if
+ we're already there or beyond.
+ """
+ logger.debug("Requested partition size for %s: %d",
+ self.mountpoint, self.size)
+
+ if not self.size:
+ return 0
+
+ requested_blocks = self.size
+
+ logger.debug("Requested blocks %d, current_blocks %d",
+ requested_blocks, current_blocks)
+
+ if requested_blocks > current_blocks:
+ return requested_blocks - current_blocks
+ else:
+ return 0
+
+ def get_rootfs_size(self, actual_rootfs_size=0):
+ """
+ Calculate the required size of rootfs taking into consideration
+ --size/--fixed-size flags as well as overhead and extra space, as
+ specified in kickstart file. Raises an error if the
+ `actual_rootfs_size` is larger than fixed-size rootfs.
+
+ """
+ if self.fixed_size:
+ rootfs_size = self.fixed_size
+ if actual_rootfs_size > rootfs_size:
+ raise WicError("Actual rootfs size (%d kB) is larger than "
+ "allowed size %d kB" %
+ (actual_rootfs_size, rootfs_size))
+ else:
+ extra_blocks = self.get_extra_block_count(actual_rootfs_size)
+ if extra_blocks < self.extra_space:
+ extra_blocks = self.extra_space
+
+ rootfs_size = actual_rootfs_size + extra_blocks
+ rootfs_size *= self.overhead_factor
+
+ logger.debug("Added %d extra blocks to %s to get to %d total blocks",
+ extra_blocks, self.mountpoint, rootfs_size)
+
+ return rootfs_size
+
+ @property
+ def disk_size(self):
+ """
+ Obtain on-disk size of partition taking into consideration
+ --size/--fixed-size options.
+
+ """
+ return self.fixed_size if self.fixed_size else self.size
+
+ def prepare(self, creator, cr_workdir, oe_builddir, rootfs_dir,
+ bootimg_dir, kernel_dir, native_sysroot):
+ """
+ Prepare content for individual partitions, depending on
+ partition command parameters.
+ """
+ if not self.source:
+ if not self.size and not self.fixed_size:
+ raise WicError("The %s partition has a size of zero. Please "
+ "specify a non-zero --size/--fixed-size for that "
+ "partition." % self.mountpoint)
+
+ if self.fstype == "swap":
+ self.prepare_swap_partition(cr_workdir, oe_builddir,
+ native_sysroot)
+ self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype)
+ else:
+ if self.fstype == 'squashfs':
+ raise WicError("It's not possible to create empty squashfs "
+ "partition '%s'" % (self.mountpoint))
+
+ rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label,
+ self.lineno, self.fstype)
+ if os.path.isfile(rootfs):
+ os.remove(rootfs)
+
+ prefix = "ext" if self.fstype.startswith("ext") else self.fstype
+ method = getattr(self, "prepare_empty_partition_" + prefix)
+ method(rootfs, oe_builddir, native_sysroot)
+ self.source_file = rootfs
+ return
+
+ plugins = PluginMgr.get_plugins('source')
+
+ if self.source not in plugins:
+ raise WicError("The '%s' --source specified for %s doesn't exist.\n\t"
+ "See 'wic list source-plugins' for a list of available"
+ " --sources.\n\tSee 'wic help source-plugins' for "
+ "details on adding a new source plugin." %
+ (self.source, self.mountpoint))
+
+ srcparams_dict = {}
+ if self.sourceparams:
+ # Split sourceparams string of the form key1=val1[,key2=val2,...]
+ # into a dict. Also accepts valueless keys i.e. without =
+ splitted = self.sourceparams.split(',')
+ srcparams_dict = dict(par.split('=') for par in splitted if par)
+
+ plugin = PluginMgr.get_plugins('source')[self.source]
+ plugin.do_configure_partition(self, srcparams_dict, creator,
+ cr_workdir, oe_builddir, bootimg_dir,
+ kernel_dir, native_sysroot)
+ plugin.do_stage_partition(self, srcparams_dict, creator,
+ cr_workdir, oe_builddir, bootimg_dir,
+ kernel_dir, native_sysroot)
+ plugin.do_prepare_partition(self, srcparams_dict, creator,
+ cr_workdir, oe_builddir, bootimg_dir,
+ kernel_dir, rootfs_dir, native_sysroot)
+
+ # further processing required Partition.size to be an integer, make
+ # sure that it is one
+ if not isinstance(self.size, int):
+ raise WicError("Partition %s internal size is not an integer. "
+ "This a bug in source plugin %s and needs to be fixed." %
+ (self.mountpoint, self.source))
+
+ if self.fixed_size and self.size > self.fixed_size:
+ raise WicError("File system image of partition %s is "
+ "larger (%d kB) than its allowed size %d kB" %
+ (self.mountpoint, self.size, self.fixed_size))
+
+ def prepare_rootfs(self, cr_workdir, oe_builddir, rootfs_dir,
+ native_sysroot):
+ """
+ Prepare content for a rootfs partition i.e. create a partition
+ and fill it from a /rootfs dir.
+
+ Currently handles ext2/3/4, btrfs and vfat.
+ """
+ p_prefix = os.environ.get("PSEUDO_PREFIX", "%s/usr" % native_sysroot)
+ p_localstatedir = os.environ.get("PSEUDO_LOCALSTATEDIR",
+ "%s/../pseudo" % rootfs_dir)
+ p_passwd = os.environ.get("PSEUDO_PASSWD", rootfs_dir)
+ p_nosymlinkexp = os.environ.get("PSEUDO_NOSYMLINKEXP", "1")
+ pseudo = "export PSEUDO_PREFIX=%s;" % p_prefix
+ pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % p_localstatedir
+ pseudo += "export PSEUDO_PASSWD=%s;" % p_passwd
+ pseudo += "export PSEUDO_NOSYMLINKEXP=%s;" % p_nosymlinkexp
+ pseudo += "%s " % get_bitbake_var("FAKEROOTCMD")
+
+ rootfs = "%s/rootfs_%s.%s.%s" % (cr_workdir, self.label,
+ self.lineno, self.fstype)
+ if os.path.isfile(rootfs):
+ os.remove(rootfs)
+
+ # Get rootfs size from bitbake variable if it's not set in .ks file
+ if not self.size:
+ # Bitbake variable ROOTFS_SIZE is calculated in
+ # Image._get_rootfs_size method from meta/lib/oe/image.py
+ # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT,
+ # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE
+ rsize_bb = get_bitbake_var('ROOTFS_SIZE')
+ if rsize_bb:
+ logger.warning('overhead-factor was specified, but size was not,'
+ ' so bitbake variables will be used for the size.'
+ ' In this case both IMAGE_OVERHEAD_FACTOR and '
+ '--overhead-factor will be applied')
+ self.size = int(round(float(rsize_bb)))
+
+ prefix = "ext" if self.fstype.startswith("ext") else self.fstype
+ method = getattr(self, "prepare_rootfs_" + prefix)
+ method(rootfs, oe_builddir, rootfs_dir, native_sysroot, pseudo)
+ self.source_file = rootfs
+
+ # get the rootfs size in the right units for kickstart (kB)
+ du_cmd = "du -Lbks %s" % rootfs
+ out = exec_cmd(du_cmd)
+ self.size = int(out.split()[0])
+
+ def prepare_rootfs_ext(self, rootfs, oe_builddir, rootfs_dir,
+ native_sysroot, pseudo):
+ """
+ Prepare content for an ext2/3/4 rootfs partition.
+ """
+ du_cmd = "du -ks %s" % rootfs_dir
+ out = exec_cmd(du_cmd)
+ actual_rootfs_size = int(out.split()[0])
+
+ rootfs_size = self.get_rootfs_size(actual_rootfs_size)
+
+ with open(rootfs, 'w') as sparse:
+ os.ftruncate(sparse.fileno(), rootfs_size * 1024)
+
+ extra_imagecmd = "-i 8192"
+
+ label_str = ""
+ if self.label:
+ label_str = "-L %s" % self.label
+
+ mkfs_cmd = "mkfs.%s -F %s %s %s -d %s" % \
+ (self.fstype, extra_imagecmd, rootfs, label_str, rootfs_dir)
+ exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
+
+ mkfs_cmd = "fsck.%s -pvfD %s" % (self.fstype, rootfs)
+ exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
+
+ def prepare_rootfs_btrfs(self, rootfs, oe_builddir, rootfs_dir,
+ native_sysroot, pseudo):
+ """
+ Prepare content for a btrfs rootfs partition.
+
+ Currently handles ext2/3/4 and btrfs.
+ """
+ du_cmd = "du -ks %s" % rootfs_dir
+ out = exec_cmd(du_cmd)
+ actual_rootfs_size = int(out.split()[0])
+
+ rootfs_size = self.get_rootfs_size(actual_rootfs_size)
+
+ with open(rootfs, 'w') as sparse:
+ os.ftruncate(sparse.fileno(), rootfs_size * 1024)
+
+ label_str = ""
+ if self.label:
+ label_str = "-L %s" % self.label
+
+ mkfs_cmd = "mkfs.%s -b %d -r %s %s %s" % \
+ (self.fstype, rootfs_size * 1024, rootfs_dir, label_str, rootfs)
+ exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
+
+ def prepare_rootfs_msdos(self, rootfs, oe_builddir, rootfs_dir,
+ native_sysroot, pseudo):
+ """
+ Prepare content for a msdos/vfat rootfs partition.
+ """
+ du_cmd = "du -bks %s" % rootfs_dir
+ out = exec_cmd(du_cmd)
+ blocks = int(out.split()[0])
+
+ rootfs_size = self.get_rootfs_size(blocks)
+
+ label_str = "-n boot"
+ if self.label:
+ label_str = "-n %s" % self.label
+
+ size_str = ""
+ if self.fstype == 'msdos':
+ size_str = "-F 16" # FAT 16
+
+ dosfs_cmd = "mkdosfs %s -S 512 %s -C %s %d" % (label_str, size_str,
+ rootfs, rootfs_size)
+ exec_native_cmd(dosfs_cmd, native_sysroot)
+
+ mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir)
+ exec_native_cmd(mcopy_cmd, native_sysroot)
+
+ chmod_cmd = "chmod 644 %s" % rootfs
+ exec_cmd(chmod_cmd)
+
+ prepare_rootfs_vfat = prepare_rootfs_msdos
+
+ def prepare_rootfs_squashfs(self, rootfs, oe_builddir, rootfs_dir,
+ native_sysroot, pseudo):
+ """
+ Prepare content for a squashfs rootfs partition.
+ """
+ squashfs_cmd = "mksquashfs %s %s -noappend" % \
+ (rootfs_dir, rootfs)
+ exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo)
+
+ def prepare_empty_partition_ext(self, rootfs, oe_builddir,
+ native_sysroot):
+ """
+ Prepare an empty ext2/3/4 partition.
+ """
+ size = self.disk_size
+ with open(rootfs, 'w') as sparse:
+ os.ftruncate(sparse.fileno(), size * 1024)
+
+ extra_imagecmd = "-i 8192"
+
+ label_str = ""
+ if self.label:
+ label_str = "-L %s" % self.label
+
+ mkfs_cmd = "mkfs.%s -F %s %s %s" % \
+ (self.fstype, extra_imagecmd, label_str, rootfs)
+ exec_native_cmd(mkfs_cmd, native_sysroot)
+
+ def prepare_empty_partition_btrfs(self, rootfs, oe_builddir,
+ native_sysroot):
+ """
+ Prepare an empty btrfs partition.
+ """
+ size = self.disk_size
+ with open(rootfs, 'w') as sparse:
+ os.ftruncate(sparse.fileno(), size * 1024)
+
+ label_str = ""
+ if self.label:
+ label_str = "-L %s" % self.label
+
+ mkfs_cmd = "mkfs.%s -b %d %s %s" % \
+ (self.fstype, self.size * 1024, label_str, rootfs)
+ exec_native_cmd(mkfs_cmd, native_sysroot)
+
+ def prepare_empty_partition_msdos(self, rootfs, oe_builddir,
+ native_sysroot):
+ """
+ Prepare an empty vfat partition.
+ """
+ blocks = self.disk_size
+
+ label_str = "-n boot"
+ if self.label:
+ label_str = "-n %s" % self.label
+
+ size_str = ""
+ if self.fstype == 'msdos':
+ size_str = "-F 16" # FAT 16
+
+ dosfs_cmd = "mkdosfs %s -S 512 %s -C %s %d" % (label_str, size_str,
+ rootfs, blocks)
+ exec_native_cmd(dosfs_cmd, native_sysroot)
+
+ chmod_cmd = "chmod 644 %s" % rootfs
+ exec_cmd(chmod_cmd)
+
+ prepare_empty_partition_vfat = prepare_empty_partition_msdos
+
+ def prepare_swap_partition(self, cr_workdir, oe_builddir, native_sysroot):
+ """
+ Prepare a swap partition.
+ """
+ path = "%s/fs.%s" % (cr_workdir, self.fstype)
+
+ with open(path, 'w') as sparse:
+ os.ftruncate(sparse.fileno(), self.size * 1024)
+
+ import uuid
+ label_str = ""
+ if self.label:
+ label_str = "-L %s" % self.label
+ mkswap_cmd = "mkswap %s -U %s %s" % (label_str, str(uuid.uuid1()), path)
+ exec_native_cmd(mkswap_cmd, native_sysroot)
diff --git a/scripts/lib/wic/plugin.py b/scripts/lib/wic/plugin.py
deleted file mode 100644
index 3acd5b480e..0000000000
--- a/scripts/lib/wic/plugin.py
+++ /dev/null
@@ -1,156 +0,0 @@
-#!/usr/bin/env python -tt
-#
-# Copyright (c) 2011 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-import os, sys
-
-from wic import msger
-from wic import pluginbase
-from wic.utils import errors
-from wic.utils.oe.misc import *
-
-__ALL__ = ['PluginMgr', 'pluginmgr']
-
-PLUGIN_TYPES = ["imager", "source"]
-
-PLUGIN_DIR = "/lib/wic/plugins" # relative to scripts
-SCRIPTS_PLUGIN_DIR = "scripts" + PLUGIN_DIR
-
-class PluginMgr(object):
- plugin_dirs = {}
-
- # make the manager class as singleton
- _instance = None
- def __new__(cls, *args, **kwargs):
- if not cls._instance:
- cls._instance = super(PluginMgr, cls).__new__(cls, *args, **kwargs)
-
- return cls._instance
-
- def __init__(self):
- wic_path = os.path.dirname(__file__)
- eos = wic_path.rfind('scripts') + len('scripts')
- scripts_path = wic_path[:eos]
- self.scripts_path = scripts_path
- self.plugin_dir = scripts_path + PLUGIN_DIR
- self.layers_path = None
-
- def _build_plugin_dir_list(self, dl, ptype):
- if self.layers_path is None:
- self.layers_path = get_bitbake_var("BBLAYERS")
- layer_dirs = []
-
- if self.layers_path is not None:
- for layer_path in self.layers_path.split():
- path = os.path.join(layer_path, SCRIPTS_PLUGIN_DIR, ptype)
- layer_dirs.append(path)
-
- path = os.path.join(dl, ptype)
- layer_dirs.append(path)
-
- return layer_dirs
-
- def append_dirs(self, dirs):
- for path in dirs:
- self._add_plugindir(path)
-
- # load all the plugins AGAIN
- self._load_all()
-
- def _add_plugindir(self, path):
- path = os.path.abspath(os.path.expanduser(path))
-
- if not os.path.isdir(path):
- msger.debug("Plugin dir is not a directory or does not exist: %s"\
- % path)
- return
-
- if path not in self.plugin_dirs:
- self.plugin_dirs[path] = False
- # the value True/False means "loaded"
-
- def _load_all(self):
- for (pdir, loaded) in self.plugin_dirs.iteritems():
- if loaded: continue
-
- sys.path.insert(0, pdir)
- for mod in [x[:-3] for x in os.listdir(pdir) if x.endswith(".py")]:
- if mod and mod != '__init__':
- if mod in sys.modules:
- #self.plugin_dirs[pdir] = True
- msger.warning("Module %s already exists, skip" % mod)
- else:
- try:
- pymod = __import__(mod)
- self.plugin_dirs[pdir] = True
- msger.debug("Plugin module %s:%s imported"\
- % (mod, pymod.__file__))
- except ImportError, err:
- msg = 'Failed to load plugin %s/%s: %s' \
- % (os.path.basename(pdir), mod, err)
- msger.warning(msg)
-
- del(sys.path[0])
-
- def get_plugins(self, ptype):
- """ the return value is dict of name:class pairs """
-
- if ptype not in PLUGIN_TYPES:
- raise errors.CreatorError('%s is not valid plugin type' % ptype)
-
- plugins_dir = self._build_plugin_dir_list(self.plugin_dir, ptype)
-
- self.append_dirs(plugins_dir)
-
- return pluginbase.get_plugins(ptype)
-
- def get_source_plugins(self):
- """
- Return list of available source plugins.
- """
- plugins_dir = self._build_plugin_dir_list(self.plugin_dir, 'source')
-
- self.append_dirs(plugins_dir)
-
- plugins = []
-
- for _source_name, klass in self.get_plugins('source').iteritems():
- plugins.append(_source_name)
-
- return plugins
-
-
- def get_source_plugin_methods(self, source_name, methods):
- """
- The methods param is a dict with the method names to find. On
- return, the dict values will be filled in with pointers to the
- corresponding methods. If one or more methods are not found,
- None is returned.
- """
- return_methods = None
- for _source_name, klass in self.get_plugins('source').iteritems():
- if _source_name == source_name:
- for _method_name in methods.keys():
- if not hasattr(klass, _method_name):
- msger.warning("Unimplemented %s source interface for: %s"\
- % (_method_name, _source_name))
- return None
- func = getattr(klass, _method_name)
- methods[_method_name] = func
- return_methods = methods
- return return_methods
-
-pluginmgr = PluginMgr()
diff --git a/scripts/lib/wic/pluginbase.py b/scripts/lib/wic/pluginbase.py
index e3de9bacb8..fb3d179c2d 100644
--- a/scripts/lib/wic/pluginbase.py
+++ b/scripts/lib/wic/pluginbase.py
@@ -15,37 +15,74 @@
# with this program; if not, write to the Free Software Foundation, Inc., 59
# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+__all__ = ['ImagerPlugin', 'SourcePlugin']
+
import os
-import shutil
-from wic import msger
-from wic.utils import errors
+import logging
+
+from collections import defaultdict
+from importlib.machinery import SourceFileLoader
-class _Plugin(object):
- class __metaclass__(type):
- def __init__(cls, name, bases, attrs):
- if not hasattr(cls, 'plugins'):
- cls.plugins = {}
+from wic import WicError
+from wic.utils.misc import get_bitbake_var
- elif 'wic_plugin_type' in attrs:
- if attrs['wic_plugin_type'] not in cls.plugins:
- cls.plugins[attrs['wic_plugin_type']] = {}
+PLUGIN_TYPES = ["imager", "source"]
- elif hasattr(cls, 'wic_plugin_type') and 'name' in attrs:
- cls.plugins[cls.wic_plugin_type][attrs['name']] = cls
+SCRIPTS_PLUGIN_DIR = "scripts/lib/wic/plugins"
- def show_plugins(cls):
- for cls in cls.plugins[cls.wic_plugin_type]:
- print cls
+logger = logging.getLogger('wic')
- def get_plugins(cls):
- return cls.plugins
+PLUGINS = defaultdict(dict)
+class PluginMgr:
+ _plugin_dirs = []
-class ImagerPlugin(_Plugin):
+ @classmethod
+ def get_plugins(cls, ptype):
+ """Get dictionary of <plugin_name>:<class> pairs."""
+ if ptype not in PLUGIN_TYPES:
+ raise WicError('%s is not valid plugin type' % ptype)
+
+ # collect plugin directories
+ if not cls._plugin_dirs:
+ cls._plugin_dirs = [os.path.join(os.path.dirname(__file__), 'plugins')]
+ layers = get_bitbake_var("BBLAYERS") or ''
+ for layer_path in layers.split():
+ path = os.path.join(layer_path, SCRIPTS_PLUGIN_DIR)
+ path = os.path.abspath(os.path.expanduser(path))
+ if path not in cls._plugin_dirs and os.path.isdir(path):
+ cls._plugin_dirs.insert(0, path)
+
+ if ptype not in PLUGINS:
+ # load all ptype plugins
+ for pdir in cls._plugin_dirs:
+ ppath = os.path.join(pdir, ptype)
+ if os.path.isdir(ppath):
+ for fname in os.listdir(ppath):
+ if fname.endswith('.py'):
+ mname = fname[:-3]
+ mpath = os.path.join(ppath, fname)
+ logger.debug("loading plugin module %s", mpath)
+ SourceFileLoader(mname, mpath).load_module()
+
+ return PLUGINS.get(ptype)
+
+class PluginMeta(type):
+ def __new__(cls, name, bases, attrs):
+ class_type = type.__new__(cls, name, bases, attrs)
+ if 'name' in attrs:
+ PLUGINS[class_type.wic_plugin_type][attrs['name']] = class_type
+
+ return class_type
+
+class ImagerPlugin(metaclass=PluginMeta):
wic_plugin_type = "imager"
+ def do_create(self):
+ raise WicError("Method %s.do_create is not implemented" %
+ self.__class__.__name__)
-class SourcePlugin(_Plugin):
+class SourcePlugin(metaclass=PluginMeta):
wic_plugin_type = "source"
"""
The methods that can be implemented by --source plugins.
@@ -54,17 +91,17 @@ class SourcePlugin(_Plugin):
"""
@classmethod
- def do_install_disk(self, disk, disk_name, cr, workdir, oe_builddir,
+ def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir,
bootimg_dir, kernel_dir, native_sysroot):
"""
Called after all partitions have been prepared and assembled into a
disk image. This provides a hook to allow finalization of a
disk image e.g. to write an MBR to it.
"""
- msger.debug("SourcePlugin: do_install_disk: disk: %s" % disk_name)
+ logger.debug("SourcePlugin: do_install_disk: disk: %s", disk_name)
@classmethod
- def do_stage_partition(self, part, source_params, cr, cr_workdir,
+ def do_stage_partition(cls, part, source_params, creator, cr_workdir,
oe_builddir, bootimg_dir, kernel_dir,
native_sysroot):
"""
@@ -78,10 +115,10 @@ class SourcePlugin(_Plugin):
Not that get_bitbake_var() allows you to acces non-standard
variables that you might want to use for this.
"""
- msger.debug("SourcePlugin: do_stage_partition: part: %s" % part)
+ logger.debug("SourcePlugin: do_stage_partition: part: %s", part)
@classmethod
- def do_configure_partition(self, part, source_params, cr, cr_workdir,
+ def do_configure_partition(cls, part, source_params, creator, cr_workdir,
oe_builddir, bootimg_dir, kernel_dir,
native_sysroot):
"""
@@ -89,23 +126,15 @@ class SourcePlugin(_Plugin):
custom configuration files for a partition, for example
syslinux or grub config files.
"""
- msger.debug("SourcePlugin: do_configure_partition: part: %s" % part)
+ logger.debug("SourcePlugin: do_configure_partition: part: %s", part)
@classmethod
- def do_prepare_partition(self, part, source_params, cr, cr_workdir,
+ def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
oe_builddir, bootimg_dir, kernel_dir, rootfs_dir,
native_sysroot):
"""
Called to do the actual content population for a partition i.e. it
'prepares' the partition to be incorporated into the image.
"""
- msger.debug("SourcePlugin: do_prepare_partition: part: %s" % part)
-
-def get_plugins(typen):
- ps = ImagerPlugin.get_plugins()
- if typen in ps:
- return ps[typen]
- else:
- return None
+ logger.debug("SourcePlugin: do_prepare_partition: part: %s", part)
-__all__ = ['ImagerPlugin', 'SourcePlugin', 'get_plugins']
diff --git a/scripts/lib/wic/plugins/imager/direct.py b/scripts/lib/wic/plugins/imager/direct.py
new file mode 100644
index 0000000000..f2e6127331
--- /dev/null
+++ b/scripts/lib/wic/plugins/imager/direct.py
@@ -0,0 +1,561 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Copyright (c) 2013, Intel Corporation.
+# All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# This implements the 'direct' imager plugin class for 'wic'
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+#
+
+import logging
+import os
+import shutil
+import tempfile
+import uuid
+
+from time import strftime
+
+from wic import WicError
+from wic.filemap import sparse_copy
+from wic.ksparser import KickStart, KickStartError
+from wic.pluginbase import PluginMgr, ImagerPlugin
+from wic.utils.misc import get_bitbake_var, exec_cmd, exec_native_cmd
+
+logger = logging.getLogger('wic')
+
+class DirectPlugin(ImagerPlugin):
+ """
+ Install a system into a file containing a partitioned disk image.
+
+ An image file is formatted with a partition table, each partition
+ created from a rootfs or other OpenEmbedded build artifact and dd'ed
+ into the virtual disk. The disk image can subsequently be dd'ed onto
+ media and used on actual hardware.
+ """
+ name = 'direct'
+
+ def __init__(self, wks_file, rootfs_dir, bootimg_dir, kernel_dir,
+ native_sysroot, oe_builddir, options):
+ try:
+ self.ks = KickStart(wks_file)
+ except KickStartError as err:
+ raise WicError(str(err))
+
+ # parse possible 'rootfs=name' items
+ self.rootfs_dir = dict(rdir.split('=') for rdir in rootfs_dir.split(' '))
+ self.bootimg_dir = bootimg_dir
+ self.kernel_dir = kernel_dir
+ self.native_sysroot = native_sysroot
+ self.oe_builddir = oe_builddir
+
+ self.outdir = options.outdir
+ self.compressor = options.compressor
+ self.bmap = options.bmap
+
+ self.name = "%s-%s" % (os.path.splitext(os.path.basename(wks_file))[0],
+ strftime("%Y%m%d%H%M"))
+ self.workdir = tempfile.mkdtemp(dir=self.outdir, prefix='tmp.wic.')
+ self._image = None
+ self.ptable_format = self.ks.bootloader.ptable
+ self.parts = self.ks.partitions
+
+ # as a convenience, set source to the boot partition source
+ # instead of forcing it to be set via bootloader --source
+ for part in self.parts:
+ if not self.ks.bootloader.source and part.mountpoint == "/boot":
+ self.ks.bootloader.source = part.source
+ break
+
+ image_path = self._full_path(self.workdir, self.parts[0].disk, "direct")
+ self._image = PartitionedImage(image_path, self.ptable_format,
+ self.parts, self.native_sysroot)
+
+ def do_create(self):
+ """
+ Plugin entry point.
+ """
+ try:
+ self.create()
+ self.assemble()
+ self.finalize()
+ self.print_info()
+ finally:
+ self.cleanup()
+
+ def _write_fstab(self, image_rootfs):
+ """overriden to generate fstab (temporarily) in rootfs. This is called
+ from _create, make sure it doesn't get called from
+ BaseImage.create()
+ """
+ if not image_rootfs:
+ return
+
+ fstab_path = image_rootfs + "/etc/fstab"
+ if not os.path.isfile(fstab_path):
+ return
+
+ with open(fstab_path) as fstab:
+ fstab_lines = fstab.readlines()
+
+ if self._update_fstab(fstab_lines, self.parts):
+ shutil.copyfile(fstab_path, fstab_path + ".orig")
+
+ with open(fstab_path, "w") as fstab:
+ fstab.writelines(fstab_lines)
+
+ return fstab_path
+
+ def _update_fstab(self, fstab_lines, parts):
+ """Assume partition order same as in wks"""
+ updated = False
+ for part in parts:
+ if not part.realnum or not part.mountpoint \
+ or part.mountpoint in ("/", "/boot"):
+ continue
+
+ # mmc device partitions are named mmcblk0p1, mmcblk0p2..
+ prefix = 'p' if part.disk.startswith('mmcblk') else ''
+ device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum)
+
+ opts = part.fsopts if part.fsopts else "defaults"
+ line = "\t".join([device_name, part.mountpoint, part.fstype,
+ opts, "0", "0"]) + "\n"
+
+ fstab_lines.append(line)
+ updated = True
+
+ return updated
+
+ def _full_path(self, path, name, extention):
+ """ Construct full file path to a file we generate. """
+ return os.path.join(path, "%s-%s.%s" % (self.name, name, extention))
+
+ #
+ # Actual implemention
+ #
+ def create(self):
+ """
+ For 'wic', we already have our build artifacts - we just create
+ filesystems from the artifacts directly and combine them into
+ a partitioned image.
+ """
+ fstab_path = self._write_fstab(self.rootfs_dir.get("ROOTFS_DIR"))
+
+ for part in self.parts:
+ # get rootfs size from bitbake variable if it's not set in .ks file
+ if not part.size:
+ # and if rootfs name is specified for the partition
+ image_name = self.rootfs_dir.get(part.rootfs_dir)
+ if image_name and os.path.sep not in image_name:
+ # Bitbake variable ROOTFS_SIZE is calculated in
+ # Image._get_rootfs_size method from meta/lib/oe/image.py
+ # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT,
+ # IMAGE_OVERHEAD_FACTOR and IMAGE_ROOTFS_EXTRA_SPACE
+ rsize_bb = get_bitbake_var('ROOTFS_SIZE', image_name)
+ if rsize_bb:
+ part.size = int(round(float(rsize_bb)))
+
+ self._image.prepare(self)
+
+ if fstab_path:
+ shutil.move(fstab_path + ".orig", fstab_path)
+
+ self._image.layout_partitions()
+ self._image.create()
+
+ def assemble(self):
+ """
+ Assemble partitions into disk image
+ """
+ self._image.assemble()
+
+ def finalize(self):
+ """
+ Finalize the disk image.
+
+ For example, prepare the image to be bootable by e.g.
+ creating and installing a bootloader configuration.
+ """
+ source_plugin = self.ks.bootloader.source
+ disk_name = self.parts[0].disk
+ if source_plugin:
+ plugin = PluginMgr.get_plugins('source')[source_plugin]
+ plugin.do_install_disk(self._image, disk_name, self, self.workdir,
+ self.oe_builddir, self.bootimg_dir,
+ self.kernel_dir, self.native_sysroot)
+
+ full_path = self._image.path
+ # Generate .bmap
+ if self.bmap:
+ logger.debug("Generating bmap file for %s", disk_name)
+ exec_native_cmd("bmaptool create %s -o %s.bmap" % (full_path, full_path),
+ self.native_sysroot)
+ # Compress the image
+ if self.compressor:
+ logger.debug("Compressing disk %s with %s", disk_name, self.compressor)
+ exec_cmd("%s %s" % (self.compressor, full_path))
+
+ def print_info(self):
+ """
+ Print the image(s) and artifacts used, for the user.
+ """
+ msg = "The new image(s) can be found here:\n"
+
+ extension = "direct" + {"gzip": ".gz",
+ "bzip2": ".bz2",
+ "xz": ".xz",
+ None: ""}.get(self.compressor)
+ full_path = self._full_path(self.outdir, self.parts[0].disk, extension)
+ msg += ' %s\n\n' % full_path
+
+ msg += 'The following build artifacts were used to create the image(s):\n'
+ for part in self.parts:
+ if part.rootfs_dir is None:
+ continue
+ if part.mountpoint == '/':
+ suffix = ':'
+ else:
+ suffix = '["%s"]:' % (part.mountpoint or part.label)
+ msg += ' ROOTFS_DIR%s%s\n' % (suffix.ljust(20), part.rootfs_dir)
+
+ msg += ' BOOTIMG_DIR: %s\n' % self.bootimg_dir
+ msg += ' KERNEL_DIR: %s\n' % self.kernel_dir
+ msg += ' NATIVE_SYSROOT: %s\n' % self.native_sysroot
+
+ logger.info(msg)
+
+ @property
+ def rootdev(self):
+ """
+ Get root device name to use as a 'root' parameter
+ in kernel command line.
+
+ Assume partition order same as in wks
+ """
+ for part in self.parts:
+ if part.mountpoint == "/":
+ if part.uuid:
+ return "PARTUUID=%s" % part.uuid
+ else:
+ suffix = 'p' if part.disk.startswith('mmcblk') else ''
+ return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum)
+
+ def cleanup(self):
+ if self._image:
+ self._image.cleanup()
+
+ # Move results to the output dir
+ if not os.path.exists(self.outdir):
+ os.makedirs(self.outdir)
+
+ for fname in os.listdir(self.workdir):
+ path = os.path.join(self.workdir, fname)
+ if os.path.isfile(path):
+ shutil.move(path, os.path.join(self.outdir, fname))
+
+ # remove work directory
+ shutil.rmtree(self.workdir, ignore_errors=True)
+
+# Overhead of the MBR partitioning scheme (just one sector)
+MBR_OVERHEAD = 1
+
+# Overhead of the GPT partitioning scheme
+GPT_OVERHEAD = 34
+
+# Size of a sector in bytes
+SECTOR_SIZE = 512
+
+class PartitionedImage():
+ """
+ Partitioned image in a file.
+ """
+
+ def __init__(self, path, ptable_format, partitions, native_sysroot=None):
+ self.path = path # Path to the image file
+ self.numpart = 0 # Number of allocated partitions
+ self.realpart = 0 # Number of partitions in the partition table
+ self.offset = 0 # Offset of next partition (in sectors)
+ self.min_size = 0 # Minimum required disk size to fit
+ # all partitions (in bytes)
+ self.ptable_format = ptable_format # Partition table format
+ # Disk system identifier
+ self.identifier = int.from_bytes(os.urandom(4), 'little')
+
+ self.partitions = partitions
+ self.partimages = []
+ # Size of a sector used in calculations
+ self.sector_size = SECTOR_SIZE
+ self.native_sysroot = native_sysroot
+
+ # calculate the real partition number, accounting for partitions not
+ # in the partition table and logical partitions
+ realnum = 0
+ for part in self.partitions:
+ if part.no_table:
+ part.realnum = 0
+ else:
+ realnum += 1
+ if self.ptable_format == 'msdos' and realnum > 3:
+ part.realnum = realnum + 1
+ continue
+ part.realnum = realnum
+
+ # generate parition UUIDs
+ for part in self.partitions:
+ if not part.uuid and part.use_uuid:
+ if self.ptable_format == 'gpt':
+ part.uuid = str(uuid.uuid4())
+ else: # msdos partition table
+ part.uuid = '%08x-%02d' % (self.identifier, part.realnum)
+
+ def prepare(self, imager):
+ """Prepare an image. Call prepare method of all image partitions."""
+ for part in self.partitions:
+ # need to create the filesystems in order to get their
+ # sizes before we can add them and do the layout.
+ part.prepare(imager, imager.workdir, imager.oe_builddir,
+ imager.rootfs_dir, imager.bootimg_dir,
+ imager.kernel_dir, imager.native_sysroot)
+
+ # Converting kB to sectors for parted
+ part.size_sec = part.disk_size * 1024 // self.sector_size
+
+ def layout_partitions(self):
+ """ Layout the partitions, meaning calculate the position of every
+ partition on the disk. The 'ptable_format' parameter defines the
+ partition table format and may be "msdos". """
+
+ logger.debug("Assigning %s partitions to disks", self.ptable_format)
+
+ # The number of primary and logical partitions. Extended partition and
+ # partitions not listed in the table are not included.
+ num_real_partitions = len([p for p in self.partitions if not p.no_table])
+
+ # Go through partitions in the order they are added in .ks file
+ for num in range(len(self.partitions)):
+ part = self.partitions[num]
+
+ if self.ptable_format == 'msdos' and part.part_type:
+ # The --part-type can also be implemented for MBR partitions,
+ # in which case it would map to the 1-byte "partition type"
+ # filed at offset 3 of the partition entry.
+ raise WicError("setting custom partition type is not " \
+ "implemented for msdos partitions")
+
+ # Get the disk where the partition is located
+ self.numpart += 1
+ if not part.no_table:
+ self.realpart += 1
+
+ if self.numpart == 1:
+ if self.ptable_format == "msdos":
+ overhead = MBR_OVERHEAD
+ elif self.ptable_format == "gpt":
+ overhead = GPT_OVERHEAD
+
+ # Skip one sector required for the partitioning scheme overhead
+ self.offset += overhead
+
+ if self.realpart > 3 and num_real_partitions > 4:
+ # Reserve a sector for EBR for every logical partition
+ # before alignment is performed.
+ if self.ptable_format == "msdos":
+ self.offset += 1
+
+ if part.align:
+ # If not first partition and we do have alignment set we need
+ # to align the partition.
+ # FIXME: This leaves a empty spaces to the disk. To fill the
+ # gaps we could enlargea the previous partition?
+
+ # Calc how much the alignment is off.
+ align_sectors = self.offset % (part.align * 1024 // self.sector_size)
+
+ if align_sectors:
+ # If partition is not aligned as required, we need
+ # to move forward to the next alignment point
+ align_sectors = (part.align * 1024 // self.sector_size) - align_sectors
+
+ logger.debug("Realignment for %s%s with %s sectors, original"
+ " offset %s, target alignment is %sK.",
+ part.disk, self.numpart, align_sectors,
+ self.offset, part.align)
+
+ # increase the offset so we actually start the partition on right alignment
+ self.offset += align_sectors
+
+ part.start = self.offset
+ self.offset += part.size_sec
+
+ part.type = 'primary'
+ if not part.no_table:
+ part.num = self.realpart
+ else:
+ part.num = 0
+
+ if self.ptable_format == "msdos":
+ # only count the partitions that are in partition table
+ if num_real_partitions > 4:
+ if self.realpart > 3:
+ part.type = 'logical'
+ part.num = self.realpart + 1
+
+ logger.debug("Assigned %s to %s%d, sectors range %d-%d size %d "
+ "sectors (%d bytes).", part.mountpoint, part.disk,
+ part.num, part.start, self.offset - 1, part.size_sec,
+ part.size_sec * self.sector_size)
+
+ # Once all the partitions have been layed out, we can calculate the
+ # minumim disk size
+ self.min_size = self.offset
+ if self.ptable_format == "gpt":
+ self.min_size += GPT_OVERHEAD
+
+ self.min_size *= self.sector_size
+
+ def _create_partition(self, device, parttype, fstype, start, size):
+ """ Create a partition on an image described by the 'device' object. """
+
+ # Start is included to the size so we need to substract one from the end.
+ end = start + size - 1
+ logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors",
+ parttype, start, end, size)
+
+ cmd = "parted -s %s unit s mkpart %s" % (device, parttype)
+ if fstype:
+ cmd += " %s" % fstype
+ cmd += " %d %d" % (start, end)
+
+ return exec_native_cmd(cmd, self.native_sysroot)
+
+ def create(self):
+ logger.debug("Creating sparse file %s", self.path)
+ with open(self.path, 'w') as sparse:
+ os.ftruncate(sparse.fileno(), self.min_size)
+
+ logger.debug("Initializing partition table for %s", self.path)
+ exec_native_cmd("parted -s %s mklabel %s" %
+ (self.path, self.ptable_format), self.native_sysroot)
+
+ logger.debug("Set disk identifier %x", self.identifier)
+ with open(self.path, 'r+b') as img:
+ img.seek(0x1B8)
+ img.write(self.identifier.to_bytes(4, 'little'))
+
+ logger.debug("Creating partitions")
+
+ for part in self.partitions:
+ if part.num == 0:
+ continue
+
+ if self.ptable_format == "msdos" and part.num == 5:
+ # Create an extended partition (note: extended
+ # partition is described in MBR and contains all
+ # logical partitions). The logical partitions save a
+ # sector for an EBR just before the start of a
+ # partition. The extended partition must start one
+ # sector before the start of the first logical
+ # partition. This way the first EBR is inside of the
+ # extended partition. Since the extended partitions
+ # starts a sector before the first logical partition,
+ # add a sector at the back, so that there is enough
+ # room for all logical partitions.
+ self._create_partition(self.path, "extended",
+ None, part.start - 1,
+ self.offset - part.start + 1)
+
+ if part.fstype == "swap":
+ parted_fs_type = "linux-swap"
+ elif part.fstype == "vfat":
+ parted_fs_type = "fat32"
+ elif part.fstype == "msdos":
+ parted_fs_type = "fat16"
+ if not part.system_id:
+ part.system_id = '0x6' # FAT16
+ else:
+ # Type for ext2/ext3/ext4/btrfs
+ parted_fs_type = "ext2"
+
+ # Boot ROM of OMAP boards require vfat boot partition to have an
+ # even number of sectors.
+ if part.mountpoint == "/boot" and part.fstype in ["vfat", "msdos"] \
+ and part.size_sec % 2:
+ logger.debug("Subtracting one sector from '%s' partition to "
+ "get even number of sectors for the partition",
+ part.mountpoint)
+ part.size_sec -= 1
+
+ self._create_partition(self.path, part.type,
+ parted_fs_type, part.start, part.size_sec)
+
+ if part.part_type:
+ logger.debug("partition %d: set type UID to %s",
+ part.num, part.part_type)
+ exec_native_cmd("sgdisk --typecode=%d:%s %s" % \
+ (part.num, part.part_type,
+ self.path), self.native_sysroot)
+
+ if part.uuid and self.ptable_format == "gpt":
+ logger.debug("partition %d: set UUID to %s",
+ part.num, part.uuid)
+ exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \
+ (part.num, part.uuid, self.path),
+ self.native_sysroot)
+
+ if part.label and self.ptable_format == "gpt":
+ logger.debug("partition %d: set name to %s",
+ part.num, part.label)
+ exec_native_cmd("parted -s %s name %d %s" % \
+ (self.path, part.num, part.label),
+ self.native_sysroot)
+
+ if part.active:
+ flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot"
+ logger.debug("Set '%s' flag for partition '%s' on disk '%s'",
+ flag_name, part.num, self.path)
+ exec_native_cmd("parted -s %s set %d %s on" % \
+ (self.path, part.num, flag_name),
+ self.native_sysroot)
+ if part.system_id:
+ exec_native_cmd("sfdisk --part-type %s %s %s" % \
+ (self.path, part.num, part.system_id),
+ self.native_sysroot)
+
+ def cleanup(self):
+ # remove partition images
+ for image in set(self.partimages):
+ os.remove(image)
+
+ def assemble(self):
+ logger.debug("Installing partitions")
+
+ for part in self.partitions:
+ source = part.source_file
+ if source:
+ # install source_file contents into a partition
+ sparse_copy(source, self.path, part.start * self.sector_size)
+
+ logger.debug("Installed %s in partition %d, sectors %d-%d, "
+ "size %d sectors", source, part.num, part.start,
+ part.start + part.size_sec - 1, part.size_sec)
+
+ partimage = self.path + '.p%d' % part.num
+ os.rename(source, partimage)
+ self.partimages.append(partimage)
diff --git a/scripts/lib/wic/plugins/imager/direct_plugin.py b/scripts/lib/wic/plugins/imager/direct_plugin.py
deleted file mode 100644
index 5601c3f1c9..0000000000
--- a/scripts/lib/wic/plugins/imager/direct_plugin.py
+++ /dev/null
@@ -1,98 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (c) 2013, Intel Corporation.
-# All rights reserved.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# DESCRIPTION
-# This implements the 'direct' imager plugin class for 'wic'
-#
-# AUTHORS
-# Tom Zanussi <tom.zanussi (at] linux.intel.com>
-#
-
-import os
-import shutil
-import re
-import tempfile
-
-from wic import msger
-from wic.utils import misc, fs_related, errors, runner, cmdln
-from wic.conf import configmgr
-from wic.plugin import pluginmgr
-
-import wic.imager.direct as direct
-from wic.pluginbase import ImagerPlugin
-
-class DirectPlugin(ImagerPlugin):
- name = 'direct'
-
- @classmethod
- def __rootfs_dir_to_dict(self, rootfs_dirs):
- """
- Gets a string that contain 'connection=dir' splitted by
- space and return a dict
- """
- krootfs_dir = {}
- for rootfs_dir in rootfs_dirs.split(' '):
- k, v = rootfs_dir.split('=')
- krootfs_dir[k] = v
-
- return krootfs_dir
-
- @classmethod
- def do_create(self, subcmd, opts, *args):
- """
- Create direct image, called from creator as 'direct' cmd
- """
- if len(args) != 7:
- raise errors.Usage("Extra arguments given")
-
- native_sysroot = args[0]
- kernel_dir = args[1]
- bootimg_dir = args[2]
- rootfs_dir = args[3]
-
- creatoropts = configmgr.create
- ksconf = args[4]
-
- image_output_dir = args[5]
- oe_builddir = args[6]
-
- krootfs_dir = self.__rootfs_dir_to_dict(rootfs_dir)
-
- configmgr._ksconf = ksconf
-
- creator = direct.DirectImageCreator(oe_builddir,
- image_output_dir,
- krootfs_dir,
- bootimg_dir,
- kernel_dir,
- native_sysroot,
- creatoropts)
-
- try:
- creator.create()
- creator.assemble()
- creator.finalize()
- creator.print_outimage_info()
-
- except errors.CreatorError:
- raise
- finally:
- creator.cleanup()
-
- return 0
diff --git a/scripts/lib/wic/plugins/source/bootimg-efi.py b/scripts/lib/wic/plugins/source/bootimg-efi.py
index ee57881e90..9879cb9fce 100644
--- a/scripts/lib/wic/plugins/source/bootimg-efi.py
+++ b/scripts/lib/wic/plugins/source/bootimg-efi.py
@@ -24,69 +24,70 @@
# Tom Zanussi <tom.zanussi (at] linux.intel.com>
#
+import logging
import os
import shutil
-import re
-import tempfile
-
-from wic import kickstart, msger
-from wic.utils import misc, fs_related, errors, runner, cmdln
-from wic.conf import configmgr
-from wic.plugin import pluginmgr
-import wic.imager.direct as direct
+
+from wic import WicError
+from wic.engine import get_custom_config
from wic.pluginbase import SourcePlugin
-from wic.utils.oe.misc import *
-from wic.imager.direct import DirectImageCreator
+from wic.utils.misc import (exec_cmd, exec_native_cmd, get_bitbake_var,
+ BOOTDD_EXTRA_SPACE)
+
+logger = logging.getLogger('wic')
class BootimgEFIPlugin(SourcePlugin):
+ """
+ Create EFI boot partition.
+ This plugin supports GRUB 2 and systemd-boot bootloaders.
+ """
+
name = 'bootimg-efi'
@classmethod
- def do_configure_grubefi(self, hdddir, cr, cr_workdir):
+ def do_configure_grubefi(cls, creator, cr_workdir):
"""
Create loader-specific (grub-efi) config
"""
- splash = os.path.join(cr_workdir, "/EFI/boot/splash.jpg")
- if os.path.exists(splash):
- splashline = "menu background splash.jpg"
- else:
- splashline = ""
-
- (rootdev, root_part_uuid) = cr._get_boot_config()
- options = cr.ks.handler.bootloader.appendLine
+ configfile = creator.ks.bootloader.configfile
+ custom_cfg = None
+ if configfile:
+ custom_cfg = get_custom_config(configfile)
+ if custom_cfg:
+ # Use a custom configuration for grub
+ grubefi_conf = custom_cfg
+ logger.debug("Using custom configuration file "
+ "%s for grub.cfg", configfile)
+ else:
+ raise WicError("configfile is specified but failed to "
+ "get it from %s." % configfile)
- grubefi_conf = ""
- grubefi_conf += "serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1\n"
- grubefi_conf += "default=boot\n"
- timeout = kickstart.get_timeout(cr.ks)
- if not timeout:
- timeout = 0
- grubefi_conf += "timeout=%s\n" % timeout
- grubefi_conf += "menuentry 'boot'{\n"
+ if not custom_cfg:
+ # Create grub configuration using parameters from wks file
+ bootloader = creator.ks.bootloader
- kernel = "/bzImage"
+ grubefi_conf = ""
+ grubefi_conf += "serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1\n"
+ grubefi_conf += "default=boot\n"
+ grubefi_conf += "timeout=%s\n" % bootloader.timeout
+ grubefi_conf += "menuentry 'boot'{\n"
- if cr._ptable_format == 'msdos':
- rootstr = rootdev
- else:
- raise ImageError("Unsupported partition table format found")
+ kernel = "/bzImage"
- grubefi_conf += "linux %s root=%s rootwait %s\n" \
- % (kernel, rootstr, options)
- grubefi_conf += "}\n"
- if splashline:
- syslinux_conf += "%s\n" % splashline
+ grubefi_conf += "linux %s root=%s rootwait %s\n" \
+ % (kernel, creator.rootdev, bootloader.append)
+ grubefi_conf += "}\n"
- msger.debug("Writing grubefi config %s/hdd/boot/EFI/BOOT/grub.cfg" \
- % cr_workdir)
+ logger.debug("Writing grubefi config %s/hdd/boot/EFI/BOOT/grub.cfg",
+ cr_workdir)
cfg = open("%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir, "w")
cfg.write(grubefi_conf)
cfg.close()
@classmethod
- def do_configure_gummiboot(self, hdddir, cr, cr_workdir):
+ def do_configure_systemdboot(cls, hdddir, creator, cr_workdir, source_params):
"""
- Create loader-specific (gummiboot) config
+ Create loader-specific systemd-boot/gummiboot config
"""
install_cmd = "install -d %s/loader" % hdddir
exec_cmd(install_cmd)
@@ -94,70 +95,89 @@ class BootimgEFIPlugin(SourcePlugin):
install_cmd = "install -d %s/loader/entries" % hdddir
exec_cmd(install_cmd)
- (rootdev, root_part_uuid) = cr._get_boot_config()
- options = cr.ks.handler.bootloader.appendLine
-
- timeout = kickstart.get_timeout(cr.ks)
- if not timeout:
- timeout = 0
+ bootloader = creator.ks.bootloader
loader_conf = ""
loader_conf += "default boot\n"
- loader_conf += "timeout %d\n" % timeout
+ loader_conf += "timeout %d\n" % bootloader.timeout
+
+ initrd = source_params.get('initrd')
+
+ if initrd:
+ # obviously we need to have a common common deploy var
+ bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+ if not bootimg_dir:
+ raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
+
+ cp_cmd = "cp %s/%s %s" % (bootimg_dir, initrd, hdddir)
+ exec_cmd(cp_cmd, True)
+ else:
+ logger.debug("Ignoring missing initrd")
- msger.debug("Writing gummiboot config %s/hdd/boot/loader/loader.conf" \
- % cr_workdir)
+ logger.debug("Writing systemd-boot config "
+ "%s/hdd/boot/loader/loader.conf", cr_workdir)
cfg = open("%s/hdd/boot/loader/loader.conf" % cr_workdir, "w")
cfg.write(loader_conf)
cfg.close()
- kernel = "/bzImage"
+ configfile = creator.ks.bootloader.configfile
+ custom_cfg = None
+ if configfile:
+ custom_cfg = get_custom_config(configfile)
+ if custom_cfg:
+ # Use a custom configuration for systemd-boot
+ boot_conf = custom_cfg
+ logger.debug("Using custom configuration file "
+ "%s for systemd-boots's boot.conf", configfile)
+ else:
+ raise WicError("configfile is specified but failed to "
+ "get it from %s.", configfile)
- if cr._ptable_format == 'msdos':
- rootstr = rootdev
- else:
- raise ImageError("Unsupported partition table format found")
+ if not custom_cfg:
+ # Create systemd-boot configuration using parameters from wks file
+ kernel = "/bzImage"
- boot_conf = ""
- boot_conf += "title boot\n"
- boot_conf += "linux %s\n" % kernel
- boot_conf += "options LABEL=Boot root=%s %s\n" \
- % (rootstr, options)
+ boot_conf = ""
+ boot_conf += "title boot\n"
+ boot_conf += "linux %s\n" % kernel
+ boot_conf += "options LABEL=Boot root=%s %s\n" % \
+ (creator.rootdev, bootloader.append)
- msger.debug("Writing gummiboot config %s/hdd/boot/loader/entries/boot.conf" \
- % cr_workdir)
+ if initrd:
+ boot_conf += "initrd /%s\n" % initrd
+
+ logger.debug("Writing systemd-boot config "
+ "%s/hdd/boot/loader/entries/boot.conf", cr_workdir)
cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w")
cfg.write(boot_conf)
cfg.close()
@classmethod
- def do_configure_partition(self, part, source_params, cr, cr_workdir,
+ def do_configure_partition(cls, part, source_params, creator, cr_workdir,
oe_builddir, bootimg_dir, kernel_dir,
native_sysroot):
"""
Called before do_prepare_partition(), creates loader-specific config
"""
hdddir = "%s/hdd/boot" % cr_workdir
- rm_cmd = "rm -rf %s" % cr_workdir
- exec_cmd(rm_cmd)
install_cmd = "install -d %s/EFI/BOOT" % hdddir
exec_cmd(install_cmd)
try:
if source_params['loader'] == 'grub-efi':
- self.do_configure_grubefi(hdddir, cr, cr_workdir)
- elif source_params['loader'] == 'gummiboot':
- self.do_configure_gummiboot(hdddir, cr, cr_workdir)
+ cls.do_configure_grubefi(creator, cr_workdir)
+ elif source_params['loader'] == 'systemd-boot':
+ cls.do_configure_systemdboot(hdddir, creator, cr_workdir, source_params)
else:
- msger.error("unrecognized bootimg-efi loader: %s" % source_params['loader'])
+ raise WicError("unrecognized bootimg-efi loader: %s" % source_params['loader'])
except KeyError:
- msger.error("bootimg-efi requires a loader, none specified")
+ raise WicError("bootimg-efi requires a loader, none specified")
@classmethod
- def do_prepare_partition(self, part, source_params, cr, cr_workdir,
+ def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
oe_builddir, bootimg_dir, kernel_dir,
rootfs_dir, native_sysroot):
"""
@@ -165,12 +185,10 @@ class BootimgEFIPlugin(SourcePlugin):
'prepares' the partition to be incorporated into the image.
In this case, prepare content for an EFI (grub) boot partition.
"""
- if not bootimg_dir:
- bootimg_dir = get_bitbake_var("HDDDIR")
- if not bootimg_dir:
- msger.error("Couldn't find HDDDIR, exiting\n")
- # just so the result notes display it
- cr.set_bootimg_dir(bootimg_dir)
+ if not kernel_dir:
+ kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+ if not kernel_dir:
+ raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
staging_kernel_dir = kernel_dir
@@ -180,21 +198,30 @@ class BootimgEFIPlugin(SourcePlugin):
(staging_kernel_dir, hdddir)
exec_cmd(install_cmd)
+
try:
if source_params['loader'] == 'grub-efi':
shutil.copyfile("%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir,
"%s/grub.cfg" % cr_workdir)
- cp_cmd = "cp %s/EFI/BOOT/* %s/EFI/BOOT" % (bootimg_dir, hdddir)
- exec_cmd(cp_cmd, True)
+ for mod in [x for x in os.listdir(kernel_dir) if x.startswith("grub-efi-")]:
+ cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[9:])
+ exec_cmd(cp_cmd, True)
shutil.move("%s/grub.cfg" % cr_workdir,
"%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir)
- elif source_params['loader'] == 'gummiboot':
- cp_cmd = "cp %s/EFI/BOOT/* %s/EFI/BOOT" % (bootimg_dir, hdddir)
- exec_cmd(cp_cmd, True)
+ elif source_params['loader'] == 'systemd-boot':
+ for mod in [x for x in os.listdir(kernel_dir) if x.startswith("systemd-")]:
+ cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[8:])
+ exec_cmd(cp_cmd, True)
else:
- msger.error("unrecognized bootimg-efi loader: %s" % source_params['loader'])
+ raise WicError("unrecognized bootimg-efi loader: %s" %
+ source_params['loader'])
except KeyError:
- msger.error("bootimg-efi requires a loader, none specified")
+ raise WicError("bootimg-efi requires a loader, none specified")
+
+ startup = os.path.join(kernel_dir, "startup.nsh")
+ if os.path.exists(startup):
+ cp_cmd = "cp %s %s/" % (startup, hdddir)
+ exec_cmd(cp_cmd, True)
du_cmd = "du -bks %s" % hdddir
out = exec_cmd(du_cmd)
@@ -207,14 +234,8 @@ class BootimgEFIPlugin(SourcePlugin):
blocks += extra_blocks
- msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \
- (extra_blocks, part.mountpoint, blocks))
-
- # Ensure total sectors is an integral number of sectors per
- # track or mcopy will complain. Sectors are 512 bytes, and we
- # generate images with 32 sectors per track. This calculation is
- # done in blocks, thus the mod by 16 instead of 32.
- blocks += (16 - (blocks % 16))
+ logger.debug("Added %d extra blocks to %s to get to %d total blocks",
+ extra_blocks, part.mountpoint, blocks)
# dosfs image, created by mkdosfs
bootimg = "%s/boot.img" % cr_workdir
@@ -232,5 +253,5 @@ class BootimgEFIPlugin(SourcePlugin):
out = exec_cmd(du_cmd)
bootimg_size = out.split()[0]
- part.set_size(bootimg_size)
- part.set_source_file(bootimg)
+ part.size = int(bootimg_size)
+ part.source_file = bootimg
diff --git a/scripts/lib/wic/plugins/source/bootimg-partition.py b/scripts/lib/wic/plugins/source/bootimg-partition.py
index c5eb7b8b80..13fddbd478 100644
--- a/scripts/lib/wic/plugins/source/bootimg-partition.py
+++ b/scripts/lib/wic/plugins/source/bootimg-partition.py
@@ -23,39 +23,28 @@
# Maciej Borzecki <maciej.borzecki (at] open-rnd.pl>
#
+import logging
import os
import re
-from wic import msger
-from wic.pluginbase import SourcePlugin
-from wic.utils.oe.misc import *
from glob import glob
-class BootimgPartitionPlugin(SourcePlugin):
- name = 'bootimg-partition'
+from wic import WicError
+from wic.pluginbase import SourcePlugin
+from wic.utils.misc import exec_cmd, get_bitbake_var
- @classmethod
- def do_install_disk(self, disk, disk_name, cr, workdir, oe_builddir,
- bootimg_dir, kernel_dir, native_sysroot):
- """
- Called after all partitions have been prepared and assembled into a
- disk image. Do nothing.
- """
- pass
+logger = logging.getLogger('wic')
- @classmethod
- def do_configure_partition(self, part, source_params, cr, cr_workdir,
- oe_builddir, bootimg_dir, kernel_dir,
- native_sysroot):
- """
- Called before do_prepare_partition(). Possibly prepare
- configuration files of some sort.
+class BootimgPartitionPlugin(SourcePlugin):
+ """
+ Create an image of boot partition, copying over files
+ listed in IMAGE_BOOT_FILES bitbake variable.
+ """
- """
- pass
+ name = 'bootimg-partition'
@classmethod
- def do_prepare_partition(self, part, source_params, cr, cr_workdir,
+ def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
oe_builddir, bootimg_dir, kernel_dir,
rootfs_dir, native_sysroot):
"""
@@ -66,25 +55,22 @@ class BootimgPartitionPlugin(SourcePlugin):
- copies all files listed in IMAGE_BOOT_FILES variable
"""
hdddir = "%s/boot" % cr_workdir
- rm_cmd = "rm -rf %s/boot" % cr_workdir
- exec_cmd(rm_cmd)
-
install_cmd = "install -d %s" % hdddir
exec_cmd(install_cmd)
- if not bootimg_dir:
- bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
- if not bootimg_dir:
- msger.error("Couldn't find DEPLOY_DIR_IMAGE, exiting\n")
+ if not kernel_dir:
+ kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+ if not kernel_dir:
+ raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
- msger.debug('Bootimg dir: %s' % bootimg_dir)
+ logger.debug('Kernel dir: %s', bootimg_dir)
boot_files = get_bitbake_var("IMAGE_BOOT_FILES")
if not boot_files:
- msger.error('No boot files defined, IMAGE_BOOT_FILES unset')
+ raise WicError('No boot files defined, IMAGE_BOOT_FILES unset')
- msger.debug('Boot files: %s' % boot_files)
+ logger.debug('Boot files: %s', boot_files)
# list of tuples (src_name, dst_name)
deploy_files = []
@@ -92,11 +78,11 @@ class BootimgPartitionPlugin(SourcePlugin):
if ';' in src_entry:
dst_entry = tuple(src_entry.split(';'))
if not dst_entry[0] or not dst_entry[1]:
- msger.error('Malformed boot file entry: %s' % (src_entry))
+ raise WicError('Malformed boot file entry: %s' % src_entry)
else:
dst_entry = (src_entry, src_entry)
- msger.debug('Destination entry: %r' % (dst_entry,))
+ logger.debug('Destination entry: %r', dst_entry)
deploy_files.append(dst_entry)
for deploy_entry in deploy_files:
@@ -112,27 +98,26 @@ class BootimgPartitionPlugin(SourcePlugin):
os.path.join(dst,
os.path.basename(name))
- srcs = glob(os.path.join(bootimg_dir, src))
+ srcs = glob(os.path.join(kernel_dir, src))
- msger.debug('Globbed sources: %s' % (', '.join(srcs)))
+ logger.debug('Globbed sources: %s', ', '.join(srcs))
for entry in srcs:
entry_dst_name = entry_name_fn(entry)
install_task.append((entry,
os.path.join(hdddir,
entry_dst_name)))
else:
- install_task = [(os.path.join(bootimg_dir, src),
+ install_task = [(os.path.join(kernel_dir, src),
os.path.join(hdddir, dst))]
for task in install_task:
src_path, dst_path = task
- msger.debug('Install %s as %s' % (os.path.basename(src_path),
- dst_path))
+ logger.debug('Install %s as %s',
+ os.path.basename(src_path), dst_path)
install_cmd = "install -m 0644 -D %s %s" \
% (src_path, dst_path)
exec_cmd(install_cmd)
- msger.debug('Prepare boot partition using rootfs in %s' % (hdddir))
+ logger.debug('Prepare boot partition using rootfs in %s', hdddir)
part.prepare_rootfs(cr_workdir, oe_builddir, hdddir,
native_sysroot)
-
diff --git a/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/scripts/lib/wic/plugins/source/bootimg-pcbios.py
index c4786a6e0e..5890c1267b 100644
--- a/scripts/lib/wic/plugins/source/bootimg-pcbios.py
+++ b/scripts/lib/wic/plugins/source/bootimg-pcbios.py
@@ -24,102 +24,126 @@
# Tom Zanussi <tom.zanussi (at] linux.intel.com>
#
+import logging
import os
-import shutil
-import re
-import tempfile
-
-from wic import kickstart, msger
-from wic.utils import misc, fs_related, errors, runner, cmdln
-from wic.conf import configmgr
-from wic.plugin import pluginmgr
-import wic.imager.direct as direct
+
+from wic import WicError
+from wic.engine import get_custom_config
+from wic.utils import runner
from wic.pluginbase import SourcePlugin
-from wic.utils.oe.misc import *
-from wic.imager.direct import DirectImageCreator
+from wic.utils.misc import (exec_cmd, exec_native_cmd,
+ get_bitbake_var, BOOTDD_EXTRA_SPACE)
+
+logger = logging.getLogger('wic')
class BootimgPcbiosPlugin(SourcePlugin):
+ """
+ Create MBR boot partition and install syslinux on it.
+ """
+
name = 'bootimg-pcbios'
@classmethod
- def do_install_disk(self, disk, disk_name, cr, workdir, oe_builddir,
+ def _get_bootimg_dir(cls, bootimg_dir, dirname):
+ """
+ Check if dirname exists in default bootimg_dir or
+ in wic-tools STAGING_DIR.
+ """
+ for result in (bootimg_dir, get_bitbake_var("STAGING_DATADIR", "wic-tools")):
+ if os.path.exists("%s/%s" % (result, dirname)):
+ return result
+
+ raise WicError("Couldn't find correct bootimg_dir, exiting")
+
+ @classmethod
+ def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir,
bootimg_dir, kernel_dir, native_sysroot):
"""
Called after all partitions have been prepared and assembled into a
disk image. In this case, we install the MBR.
"""
+ bootimg_dir = cls._get_bootimg_dir(bootimg_dir, 'syslinux')
mbrfile = "%s/syslinux/" % bootimg_dir
- if cr._ptable_format == 'msdos':
+ if creator.ptable_format == 'msdos':
mbrfile += "mbr.bin"
+ elif creator.ptable_format == 'gpt':
+ mbrfile += "gptmbr.bin"
+ else:
+ raise WicError("Unsupported partition table: %s" %
+ creator.ptable_format)
if not os.path.exists(mbrfile):
- msger.error("Couldn't find %s. If using the -e option, do you have the right MACHINE set in local.conf? If not, is the bootimg_dir path correct?" % mbrfile)
+ raise WicError("Couldn't find %s. If using the -e option, do you "
+ "have the right MACHINE set in local.conf? If not, "
+ "is the bootimg_dir path correct?" % mbrfile)
- full_path = cr._full_path(workdir, disk_name, "direct")
- msger.debug("Installing MBR on disk %s as %s with size %s bytes" \
- % (disk_name, full_path, disk['min_size']))
+ full_path = creator._full_path(workdir, disk_name, "direct")
+ logger.debug("Installing MBR on disk %s as %s with size %s bytes",
+ disk_name, full_path, disk.min_size)
- rc = runner.show(['dd', 'if=%s' % mbrfile,
- 'of=%s' % full_path, 'conv=notrunc'])
- if rc != 0:
- raise ImageError("Unable to set MBR to %s" % full_path)
+ dd_cmd = "dd if=%s of=%s conv=notrunc" % (mbrfile, full_path)
+ exec_cmd(dd_cmd, native_sysroot)
@classmethod
- def do_configure_partition(self, part, source_params, cr, cr_workdir,
+ def do_configure_partition(cls, part, source_params, creator, cr_workdir,
oe_builddir, bootimg_dir, kernel_dir,
native_sysroot):
"""
Called before do_prepare_partition(), creates syslinux config
"""
hdddir = "%s/hdd/boot" % cr_workdir
- rm_cmd = "rm -rf " + cr_workdir
- exec_cmd(rm_cmd)
install_cmd = "install -d %s" % hdddir
exec_cmd(install_cmd)
- splash = os.path.join(cr_workdir, "/hdd/boot/splash.jpg")
- if os.path.exists(splash):
- splashline = "menu background splash.jpg"
- else:
- splashline = ""
-
- (rootdev, root_part_uuid) = cr._get_boot_config()
- options = cr.ks.handler.bootloader.appendLine
-
- syslinux_conf = ""
- syslinux_conf += "PROMPT 0\n"
- timeout = kickstart.get_timeout(cr.ks)
- if not timeout:
- timeout = 0
- syslinux_conf += "TIMEOUT " + str(timeout) + "\n"
- syslinux_conf += "\n"
- syslinux_conf += "ALLOWOPTIONS 1\n"
- syslinux_conf += "SERIAL 0 115200\n"
- syslinux_conf += "\n"
- if splashline:
- syslinux_conf += "%s\n" % splashline
- syslinux_conf += "DEFAULT boot\n"
- syslinux_conf += "LABEL boot\n"
-
- kernel = "/vmlinuz"
- syslinux_conf += "KERNEL " + kernel + "\n"
-
- if cr._ptable_format == 'msdos':
- rootstr = rootdev
- else:
- raise ImageError("Unsupported partition table format found")
-
- syslinux_conf += "APPEND label=boot root=%s %s\n" % (rootstr, options)
-
- msger.debug("Writing syslinux config %s/hdd/boot/syslinux.cfg" \
- % cr_workdir)
+ bootloader = creator.ks.bootloader
+
+ custom_cfg = None
+ if bootloader.configfile:
+ custom_cfg = get_custom_config(bootloader.configfile)
+ if custom_cfg:
+ # Use a custom configuration for grub
+ syslinux_conf = custom_cfg
+ logger.debug("Using custom configuration file %s "
+ "for syslinux.cfg", bootloader.configfile)
+ else:
+ raise WicError("configfile is specified but failed to "
+ "get it from %s." % bootloader.configfile)
+
+ if not custom_cfg:
+ # Create syslinux configuration using parameters from wks file
+ splash = os.path.join(cr_workdir, "/hdd/boot/splash.jpg")
+ if os.path.exists(splash):
+ splashline = "menu background splash.jpg"
+ else:
+ splashline = ""
+
+ syslinux_conf = ""
+ syslinux_conf += "PROMPT 0\n"
+ syslinux_conf += "TIMEOUT " + str(bootloader.timeout) + "\n"
+ syslinux_conf += "\n"
+ syslinux_conf += "ALLOWOPTIONS 1\n"
+ syslinux_conf += "SERIAL 0 115200\n"
+ syslinux_conf += "\n"
+ if splashline:
+ syslinux_conf += "%s\n" % splashline
+ syslinux_conf += "DEFAULT boot\n"
+ syslinux_conf += "LABEL boot\n"
+
+ kernel = "/vmlinuz"
+ syslinux_conf += "KERNEL " + kernel + "\n"
+
+ syslinux_conf += "APPEND label=boot root=%s %s\n" % \
+ (creator.rootdev, bootloader.append)
+
+ logger.debug("Writing syslinux config %s/hdd/boot/syslinux.cfg",
+ cr_workdir)
cfg = open("%s/hdd/boot/syslinux.cfg" % cr_workdir, "w")
cfg.write(syslinux_conf)
cfg.close()
@classmethod
- def do_prepare_partition(self, part, source_params, cr, cr_workdir,
+ def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
oe_builddir, bootimg_dir, kernel_dir,
rootfs_dir, native_sysroot):
"""
@@ -127,33 +151,25 @@ class BootimgPcbiosPlugin(SourcePlugin):
'prepares' the partition to be incorporated into the image.
In this case, prepare content for legacy bios boot partition.
"""
- def _has_syslinux(dir):
- if dir:
- syslinux = "%s/syslinux" % dir
- if os.path.exists(syslinux):
- return True
- return False
-
- if not _has_syslinux(bootimg_dir):
- bootimg_dir = get_bitbake_var("STAGING_DATADIR")
- if not bootimg_dir:
- msger.error("Couldn't find STAGING_DATADIR, exiting\n")
- if not _has_syslinux(bootimg_dir):
- msger.error("Please build syslinux first\n")
- # just so the result notes display it
- cr.set_bootimg_dir(bootimg_dir)
+ bootimg_dir = cls._get_bootimg_dir(bootimg_dir, 'syslinux')
staging_kernel_dir = kernel_dir
hdddir = "%s/hdd/boot" % cr_workdir
- install_cmd = "install -m 0644 %s/bzImage %s/vmlinuz" \
- % (staging_kernel_dir, hdddir)
- exec_cmd(install_cmd)
+ cmds = ("install -m 0644 %s/bzImage %s/vmlinuz" %
+ (staging_kernel_dir, hdddir),
+ "install -m 444 %s/syslinux/ldlinux.sys %s/ldlinux.sys" %
+ (bootimg_dir, hdddir),
+ "install -m 0644 %s/syslinux/vesamenu.c32 %s/vesamenu.c32" %
+ (bootimg_dir, hdddir),
+ "install -m 444 %s/syslinux/libcom32.c32 %s/libcom32.c32" %
+ (bootimg_dir, hdddir),
+ "install -m 444 %s/syslinux/libutil.c32 %s/libutil.c32" %
+ (bootimg_dir, hdddir))
- install_cmd = "install -m 444 %s/syslinux/ldlinux.sys %s/ldlinux.sys" \
- % (bootimg_dir, hdddir)
- exec_cmd(install_cmd)
+ for install_cmd in cmds:
+ exec_cmd(install_cmd)
du_cmd = "du -bks %s" % hdddir
out = exec_cmd(du_cmd)
@@ -166,14 +182,8 @@ class BootimgPcbiosPlugin(SourcePlugin):
blocks += extra_blocks
- msger.debug("Added %d extra blocks to %s to get to %d total blocks" % \
- (extra_blocks, part.mountpoint, blocks))
-
- # Ensure total sectors is an integral number of sectors per
- # track or mcopy will complain. Sectors are 512 bytes, and we
- # generate images with 32 sectors per track. This calculation is
- # done in blocks, thus the mod by 16 instead of 32.
- blocks += (16 - (blocks % 16))
+ logger.debug("Added %d extra blocks to %s to get to %d total blocks",
+ extra_blocks, part.mountpoint, blocks)
# dosfs image, created by mkdosfs
bootimg = "%s/boot.img" % cr_workdir
@@ -194,7 +204,5 @@ class BootimgPcbiosPlugin(SourcePlugin):
out = exec_cmd(du_cmd)
bootimg_size = out.split()[0]
- part.set_size(bootimg_size)
- part.set_source_file(bootimg)
-
-
+ part.size = int(bootimg_size)
+ part.source_file = bootimg
diff --git a/scripts/lib/wic/plugins/source/fsimage.py b/scripts/lib/wic/plugins/source/fsimage.py
deleted file mode 100644
index 0967883afa..0000000000
--- a/scripts/lib/wic/plugins/source/fsimage.py
+++ /dev/null
@@ -1,70 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-
-import os
-import re
-
-from wic import msger
-from wic.pluginbase import SourcePlugin
-from wic.utils.oe.misc import *
-
-class FSImagePlugin(SourcePlugin):
- name = 'fsimage'
-
- @classmethod
- def do_install_disk(self, disk, disk_name, cr, workdir, oe_builddir,
- bootimg_dir, kernel_dir, native_sysroot):
- """
- Called after all partitions have been prepared and assembled into a
- disk image. Do nothing.
- """
- pass
-
- @classmethod
- def do_configure_partition(self, part, source_params, cr, cr_workdir,
- oe_builddir, bootimg_dir, kernel_dir,
- native_sysroot):
- """
- Called before do_prepare_partition(). Possibly prepare
- configuration files of some sort.
- """
- pass
-
- @classmethod
- def do_prepare_partition(self, part, source_params, cr, cr_workdir,
- oe_builddir, bootimg_dir, kernel_dir,
- rootfs_dir, native_sysroot):
- """
- Called to do the actual content population for a partition i.e. it
- 'prepares' the partition to be incorporated into the image.
- """
- if not bootimg_dir:
- bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
- if not bootimg_dir:
- msger.error("Couldn't find DEPLOY_DIR_IMAGE, exiting\n")
-
- msger.debug('Bootimg dir: %s' % bootimg_dir)
-
- if ('file' not in source_params):
- msger.error("No file specified\n")
- return
-
- src = os.path.join(bootimg_dir, source_params['file'])
-
-
- msger.debug('Preparing partition using image %s' % (src))
- part.prepare_rootfs_from_fs_image(cr_workdir, src, "")
diff --git a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
new file mode 100644
index 0000000000..1ceba62be0
--- /dev/null
+++ b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
@@ -0,0 +1,494 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# This implements the 'isoimage-isohybrid' source plugin class for 'wic'
+#
+# AUTHORS
+# Mihaly Varga <mihaly.varga (at] ni.com>
+
+import glob
+import logging
+import os
+import re
+import shutil
+
+from wic import WicError
+from wic.engine import get_custom_config
+from wic.pluginbase import SourcePlugin
+from wic.utils.misc import exec_cmd, exec_native_cmd, get_bitbake_var
+
+logger = logging.getLogger('wic')
+
+class IsoImagePlugin(SourcePlugin):
+ """
+ Create a bootable ISO image
+
+ This plugin creates a hybrid, legacy and EFI bootable ISO image. The
+ generated image can be used on optical media as well as USB media.
+
+ Legacy boot uses syslinux and EFI boot uses grub or gummiboot (not
+ implemented yet) as bootloader. The plugin creates the directories required
+ by bootloaders and populates them by creating and configuring the
+ bootloader files.
+
+ Example kickstart file:
+ part /boot --source isoimage-isohybrid --sourceparams="loader=grub-efi, \\
+ image_name= IsoImage" --ondisk cd --label LIVECD --fstype=ext2
+ bootloader --timeout=10 --append=" "
+
+ In --sourceparams "loader" specifies the bootloader used for booting in EFI
+ mode, while "image_name" specifies the name of the generated image. In the
+ example above, wic creates an ISO image named IsoImage-cd.direct (default
+ extension added by direct imeger plugin) and a file named IsoImage-cd.iso
+ """
+
+ name = 'isoimage-isohybrid'
+
+ @classmethod
+ def do_configure_syslinux(cls, creator, cr_workdir):
+ """
+ Create loader-specific (syslinux) config
+ """
+ splash = os.path.join(cr_workdir, "ISO/boot/splash.jpg")
+ if os.path.exists(splash):
+ splashline = "menu background splash.jpg"
+ else:
+ splashline = ""
+
+ bootloader = creator.ks.bootloader
+
+ syslinux_conf = ""
+ syslinux_conf += "PROMPT 0\n"
+ syslinux_conf += "TIMEOUT %s \n" % (bootloader.timeout or 10)
+ syslinux_conf += "\n"
+ syslinux_conf += "ALLOWOPTIONS 1\n"
+ syslinux_conf += "SERIAL 0 115200\n"
+ syslinux_conf += "\n"
+ if splashline:
+ syslinux_conf += "%s\n" % splashline
+ syslinux_conf += "DEFAULT boot\n"
+ syslinux_conf += "LABEL boot\n"
+
+ kernel = "/bzImage"
+ syslinux_conf += "KERNEL " + kernel + "\n"
+ syslinux_conf += "APPEND initrd=/initrd LABEL=boot %s\n" \
+ % bootloader.append
+
+ logger.debug("Writing syslinux config %s/ISO/isolinux/isolinux.cfg",
+ cr_workdir)
+
+ with open("%s/ISO/isolinux/isolinux.cfg" % cr_workdir, "w") as cfg:
+ cfg.write(syslinux_conf)
+
+ @classmethod
+ def do_configure_grubefi(cls, part, creator, cr_workdir):
+ """
+ Create loader-specific (grub-efi) config
+ """
+ configfile = creator.ks.bootloader.configfile
+ if configfile:
+ grubefi_conf = get_custom_config(configfile)
+ if grubefi_conf:
+ logger.debug("Using custom configuration file %s for grub.cfg",
+ configfile)
+ else:
+ raise WicError("configfile is specified "
+ "but failed to get it from %s", configfile)
+ else:
+ splash = os.path.join(cr_workdir, "EFI/boot/splash.jpg")
+ if os.path.exists(splash):
+ splashline = "menu background splash.jpg"
+ else:
+ splashline = ""
+
+ bootloader = creator.ks.bootloader
+
+ grubefi_conf = ""
+ grubefi_conf += "serial --unit=0 --speed=115200 --word=8 "
+ grubefi_conf += "--parity=no --stop=1\n"
+ grubefi_conf += "default=boot\n"
+ grubefi_conf += "timeout=%s\n" % (bootloader.timeout or 10)
+ grubefi_conf += "\n"
+ grubefi_conf += "search --set=root --label %s " % part.label
+ grubefi_conf += "\n"
+ grubefi_conf += "menuentry 'boot'{\n"
+
+ kernel = "/bzImage"
+
+ grubefi_conf += "linux %s rootwait %s\n" \
+ % (kernel, bootloader.append)
+ grubefi_conf += "initrd /initrd \n"
+ grubefi_conf += "}\n"
+
+ if splashline:
+ grubefi_conf += "%s\n" % splashline
+
+ logger.debug("Writing grubefi config %s/EFI/BOOT/grub.cfg", cr_workdir)
+
+ with open("%s/EFI/BOOT/grub.cfg" % cr_workdir, "w") as cfg:
+ cfg.write(grubefi_conf)
+
+ @staticmethod
+ def _build_initramfs_path(rootfs_dir, cr_workdir):
+ """
+ Create path for initramfs image
+ """
+
+ initrd = get_bitbake_var("INITRD_LIVE") or get_bitbake_var("INITRD")
+ if not initrd:
+ initrd_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+ if not initrd_dir:
+ raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting.")
+
+ image_name = get_bitbake_var("IMAGE_BASENAME")
+ if not image_name:
+ raise WicError("Couldn't find IMAGE_BASENAME, exiting.")
+
+ image_type = get_bitbake_var("INITRAMFS_FSTYPES")
+ if not image_type:
+ raise WicError("Couldn't find INITRAMFS_FSTYPES, exiting.")
+
+ target_arch = get_bitbake_var("TRANSLATED_TARGET_ARCH")
+ if not target_arch:
+ raise WicError("Couldn't find TRANSLATED_TARGET_ARCH, exiting.")
+
+ initrd = glob.glob('%s/%s*%s.%s' % (initrd_dir, image_name, target_arch, image_type))[0]
+
+ if not os.path.exists(initrd):
+ # Create initrd from rootfs directory
+ initrd = "%s/initrd.cpio.gz" % cr_workdir
+ initrd_dir = "%s/INITRD" % cr_workdir
+ shutil.copytree("%s" % rootfs_dir, \
+ "%s" % initrd_dir, symlinks=True)
+
+ if os.path.isfile("%s/init" % rootfs_dir):
+ shutil.copy2("%s/init" % rootfs_dir, "%s/init" % initrd_dir)
+ elif os.path.lexists("%s/init" % rootfs_dir):
+ os.symlink(os.readlink("%s/init" % rootfs_dir), \
+ "%s/init" % initrd_dir)
+ elif os.path.isfile("%s/sbin/init" % rootfs_dir):
+ shutil.copy2("%s/sbin/init" % rootfs_dir, \
+ "%s" % initrd_dir)
+ elif os.path.lexists("%s/sbin/init" % rootfs_dir):
+ os.symlink(os.readlink("%s/sbin/init" % rootfs_dir), \
+ "%s/init" % initrd_dir)
+ else:
+ raise WicError("Couldn't find or build initrd, exiting.")
+
+ exec_cmd("cd %s && find . | cpio -o -H newc -R +0:+0 >./initrd.cpio " \
+ % initrd_dir, as_shell=True)
+ exec_cmd("gzip -f -9 -c %s/initrd.cpio > %s" \
+ % (cr_workdir, initrd), as_shell=True)
+ shutil.rmtree(initrd_dir)
+
+ return initrd
+
+ @classmethod
+ def do_configure_partition(cls, part, source_params, creator, cr_workdir,
+ oe_builddir, bootimg_dir, kernel_dir,
+ native_sysroot):
+ """
+ Called before do_prepare_partition(), creates loader-specific config
+ """
+ isodir = "%s/ISO/" % cr_workdir
+
+ if os.path.exists(cr_workdir):
+ shutil.rmtree(cr_workdir)
+
+ install_cmd = "install -d %s " % isodir
+ exec_cmd(install_cmd)
+
+ # Overwrite the name of the created image
+ logger.debug(source_params)
+ if 'image_name' in source_params and \
+ source_params['image_name'].strip():
+ creator.name = source_params['image_name'].strip()
+ logger.debug("The name of the image is: %s", creator.name)
+
+ @classmethod
+ def do_prepare_partition(cls, part, source_params, creator, cr_workdir,
+ oe_builddir, bootimg_dir, kernel_dir,
+ rootfs_dir, native_sysroot):
+ """
+ Called to do the actual content population for a partition i.e. it
+ 'prepares' the partition to be incorporated into the image.
+ In this case, prepare content for a bootable ISO image.
+ """
+
+ isodir = "%s/ISO" % cr_workdir
+
+ if part.rootfs_dir is None:
+ if not 'ROOTFS_DIR' in rootfs_dir:
+ raise WicError("Couldn't find --rootfs-dir, exiting.")
+ rootfs_dir = rootfs_dir['ROOTFS_DIR']
+ else:
+ if part.rootfs_dir in rootfs_dir:
+ rootfs_dir = rootfs_dir[part.rootfs_dir]
+ elif part.rootfs_dir:
+ rootfs_dir = part.rootfs_dir
+ else:
+ raise WicError("Couldn't find --rootfs-dir=%s connection "
+ "or it is not a valid path, exiting." %
+ part.rootfs_dir)
+
+ if not os.path.isdir(rootfs_dir):
+ rootfs_dir = get_bitbake_var("IMAGE_ROOTFS")
+ if not os.path.isdir(rootfs_dir):
+ raise WicError("Couldn't find IMAGE_ROOTFS, exiting.")
+
+ part.rootfs_dir = rootfs_dir
+
+ # Prepare rootfs.img
+ deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+ img_iso_dir = get_bitbake_var("ISODIR")
+ rootfs_img = "%s/rootfs.img" % img_iso_dir
+ if not os.path.isfile(rootfs_img):
+ # check if rootfs.img is in deploydir
+ deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+ image_name = get_bitbake_var("IMAGE_LINK_NAME")
+ rootfs_img = "%s/%s.%s" \
+ % (deploy_dir, image_name, part.fstype)
+
+ if not os.path.isfile(rootfs_img):
+ # create image file with type specified by --fstype
+ # which contains rootfs
+ du_cmd = "du -bks %s" % rootfs_dir
+ out = exec_cmd(du_cmd)
+ part.size = int(out.split()[0])
+ part.extra_space = 0
+ part.overhead_factor = 1.2
+ part.prepare_rootfs(cr_workdir, oe_builddir, rootfs_dir, \
+ native_sysroot)
+ rootfs_img = part.source_file
+
+ install_cmd = "install -m 0644 %s %s/rootfs.img" \
+ % (rootfs_img, isodir)
+ exec_cmd(install_cmd)
+
+ # Remove the temporary file created by part.prepare_rootfs()
+ if os.path.isfile(part.source_file):
+ os.remove(part.source_file)
+
+ # Support using a different initrd other than default
+ if source_params.get('initrd'):
+ initrd = source_params['initrd']
+ if not deploy_dir:
+ raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
+ cp_cmd = "cp %s/%s %s" % (deploy_dir, initrd, cr_workdir)
+ exec_cmd(cp_cmd)
+ else:
+ # Prepare initial ramdisk
+ initrd = "%s/initrd" % deploy_dir
+ if not os.path.isfile(initrd):
+ initrd = "%s/initrd" % img_iso_dir
+ if not os.path.isfile(initrd):
+ initrd = cls._build_initramfs_path(rootfs_dir, cr_workdir)
+
+ install_cmd = "install -m 0644 %s %s/initrd" % (initrd, isodir)
+ exec_cmd(install_cmd)
+
+ # Remove the temporary file created by _build_initramfs_path function
+ if os.path.isfile("%s/initrd.cpio.gz" % cr_workdir):
+ os.remove("%s/initrd.cpio.gz" % cr_workdir)
+
+ # Install bzImage
+ install_cmd = "install -m 0644 %s/bzImage %s/bzImage" % \
+ (kernel_dir, isodir)
+ exec_cmd(install_cmd)
+
+ #Create bootloader for efi boot
+ try:
+ if source_params['loader'] == 'grub-efi':
+ # Builds grub.cfg if ISODIR didn't exist or
+ # didn't contains grub.cfg
+ bootimg_dir = img_iso_dir
+ if not os.path.exists("%s/EFI/BOOT" % bootimg_dir):
+ bootimg_dir = "%s/bootimg" % cr_workdir
+ if os.path.exists(bootimg_dir):
+ shutil.rmtree(bootimg_dir)
+ install_cmd = "install -d %s/EFI/BOOT" % bootimg_dir
+ exec_cmd(install_cmd)
+
+ if not os.path.isfile("%s/EFI/BOOT/boot.cfg" % bootimg_dir):
+ cls.do_configure_grubefi(part, creator, bootimg_dir)
+
+ # Builds bootx64.efi/bootia32.efi if ISODIR didn't exist or
+ # didn't contains it
+ target_arch = get_bitbake_var("TARGET_SYS")
+ if not target_arch:
+ raise WicError("Coludn't find target architecture")
+
+ if re.match("x86_64", target_arch):
+ grub_target = 'x86_64-efi'
+ grub_image = "bootx64.efi"
+ elif re.match('i.86', target_arch):
+ grub_target = 'i386-efi'
+ grub_image = "bootia32.efi"
+ else:
+ raise WicError("grub-efi is incompatible with target %s" %
+ target_arch)
+
+ if not os.path.isfile("%s/EFI/BOOT/%s" \
+ % (bootimg_dir, grub_image)):
+ grub_path = get_bitbake_var("STAGING_LIBDIR", "wic-tools")
+ if not grub_path:
+ raise WicError("Couldn't find STAGING_LIBDIR, exiting.")
+
+ grub_core = "%s/grub/%s" % (grub_path, grub_target)
+ if not os.path.exists(grub_core):
+ raise WicError("Please build grub-efi first")
+
+ grub_cmd = "grub-mkimage -p '/EFI/BOOT' "
+ grub_cmd += "-d %s " % grub_core
+ grub_cmd += "-O %s -o %s/EFI/BOOT/%s " \
+ % (grub_target, bootimg_dir, grub_image)
+ grub_cmd += "part_gpt part_msdos ntfs ntfscomp fat ext2 "
+ grub_cmd += "normal chain boot configfile linux multiboot "
+ grub_cmd += "search efi_gop efi_uga font gfxterm gfxmenu "
+ grub_cmd += "terminal minicmd test iorw loadenv echo help "
+ grub_cmd += "reboot serial terminfo iso9660 loopback tar "
+ grub_cmd += "memdisk ls search_fs_uuid udf btrfs xfs lvm "
+ grub_cmd += "reiserfs ata "
+ exec_native_cmd(grub_cmd, native_sysroot)
+
+ else:
+ raise WicError("unrecognized bootimg-efi loader: %s" %
+ source_params['loader'])
+ except KeyError:
+ raise WicError("bootimg-efi requires a loader, none specified")
+
+ if os.path.exists("%s/EFI/BOOT" % isodir):
+ shutil.rmtree("%s/EFI/BOOT" % isodir)
+
+ shutil.copytree(bootimg_dir+"/EFI/BOOT", isodir+"/EFI/BOOT")
+
+ # If exists, remove cr_workdir/bootimg temporary folder
+ if os.path.exists("%s/bootimg" % cr_workdir):
+ shutil.rmtree("%s/bootimg" % cr_workdir)
+
+ # Create efi.img that contains bootloader files for EFI booting
+ # if ISODIR didn't exist or didn't contains it
+ if os.path.isfile("%s/efi.img" % img_iso_dir):
+ install_cmd = "install -m 0644 %s/efi.img %s/efi.img" % \
+ (img_iso_dir, isodir)
+ exec_cmd(install_cmd)
+ else:
+ du_cmd = "du -bks %s/EFI" % isodir
+ out = exec_cmd(du_cmd)
+ blocks = int(out.split()[0])
+ # Add some extra space for file system overhead
+ blocks += 100
+ logger.debug("Added 100 extra blocks to %s to get to %d "
+ "total blocks", part.mountpoint, blocks)
+
+ # dosfs image for EFI boot
+ bootimg = "%s/efi.img" % isodir
+
+ dosfs_cmd = 'mkfs.vfat -n "EFIimg" -S 512 -C %s %d' \
+ % (bootimg, blocks)
+ exec_native_cmd(dosfs_cmd, native_sysroot)
+
+ mmd_cmd = "mmd -i %s ::/EFI" % bootimg
+ exec_native_cmd(mmd_cmd, native_sysroot)
+
+ mcopy_cmd = "mcopy -i %s -s %s/EFI/* ::/EFI/" \
+ % (bootimg, isodir)
+ exec_native_cmd(mcopy_cmd, native_sysroot)
+
+ chmod_cmd = "chmod 644 %s" % bootimg
+ exec_cmd(chmod_cmd)
+
+ # Prepare files for legacy boot
+ syslinux_dir = get_bitbake_var("STAGING_DATADIR", "wic-tools")
+ if not syslinux_dir:
+ raise WicError("Couldn't find STAGING_DATADIR, exiting.")
+
+ if os.path.exists("%s/isolinux" % isodir):
+ shutil.rmtree("%s/isolinux" % isodir)
+
+ install_cmd = "install -d %s/isolinux" % isodir
+ exec_cmd(install_cmd)
+
+ cls.do_configure_syslinux(creator, cr_workdir)
+
+ install_cmd = "install -m 444 %s/syslinux/ldlinux.sys " % syslinux_dir
+ install_cmd += "%s/isolinux/ldlinux.sys" % isodir
+ exec_cmd(install_cmd)
+
+ install_cmd = "install -m 444 %s/syslinux/isohdpfx.bin " % syslinux_dir
+ install_cmd += "%s/isolinux/isohdpfx.bin" % isodir
+ exec_cmd(install_cmd)
+
+ install_cmd = "install -m 644 %s/syslinux/isolinux.bin " % syslinux_dir
+ install_cmd += "%s/isolinux/isolinux.bin" % isodir
+ exec_cmd(install_cmd)
+
+ install_cmd = "install -m 644 %s/syslinux/ldlinux.c32 " % syslinux_dir
+ install_cmd += "%s/isolinux/ldlinux.c32" % isodir
+ exec_cmd(install_cmd)
+
+ #create ISO image
+ iso_img = "%s/tempiso_img.iso" % cr_workdir
+ iso_bootimg = "isolinux/isolinux.bin"
+ iso_bootcat = "isolinux/boot.cat"
+ efi_img = "efi.img"
+
+ mkisofs_cmd = "mkisofs -V %s " % part.label
+ mkisofs_cmd += "-o %s -U " % iso_img
+ mkisofs_cmd += "-J -joliet-long -r -iso-level 2 -b %s " % iso_bootimg
+ mkisofs_cmd += "-c %s -no-emul-boot -boot-load-size 4 " % iso_bootcat
+ mkisofs_cmd += "-boot-info-table -eltorito-alt-boot "
+ mkisofs_cmd += "-eltorito-platform 0xEF -eltorito-boot %s " % efi_img
+ mkisofs_cmd += "-no-emul-boot %s " % isodir
+
+ logger.debug("running command: %s", mkisofs_cmd)
+ exec_native_cmd(mkisofs_cmd, native_sysroot)
+
+ shutil.rmtree(isodir)
+
+ du_cmd = "du -Lbks %s" % iso_img
+ out = exec_cmd(du_cmd)
+ isoimg_size = int(out.split()[0])
+
+ part.size = isoimg_size
+ part.source_file = iso_img
+
+ @classmethod
+ def do_install_disk(cls, disk, disk_name, creator, workdir, oe_builddir,
+ bootimg_dir, kernel_dir, native_sysroot):
+ """
+ Called after all partitions have been prepared and assembled into a
+ disk image. In this case, we insert/modify the MBR using isohybrid
+ utility for booting via BIOS from disk storage devices.
+ """
+
+ iso_img = "%s.p1" % disk.path
+ full_path = creator._full_path(workdir, disk_name, "direct")
+ full_path_iso = creator._full_path(workdir, disk_name, "iso")
+
+ isohybrid_cmd = "isohybrid -u %s" % iso_img
+ logger.debug("running command: %s", isohybrid_cmd)
+ exec_native_cmd(isohybrid_cmd, native_sysroot)
+
+ # Replace the image created by direct plugin with the one created by
+ # mkisofs command. This is necessary because the iso image created by
+ # mkisofs has a very specific MBR is system area of the ISO image, and
+ # direct plugin adds and configures an another MBR.
+ logger.debug("Replaceing the image created by direct plugin\n")
+ os.remove(disk.path)
+ shutil.copy2(iso_img, full_path_iso)
+ shutil.copy2(full_path_iso, full_path)
diff --git a/scripts/lib/wic/plugins/source/rawcopy.py b/scripts/lib/wic/plugins/source/rawcopy.py
index cf6236a04f..e1c4f5e7db 100644
--- a/scripts/lib/wic/plugins/source/rawcopy.py
+++ b/scripts/lib/wic/plugins/source/rawcopy.py
@@ -15,70 +15,55 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
+import logging
import os
-import re
-from wic import msger
+from wic import WicError
from wic.pluginbase import SourcePlugin
-from wic.utils.oe.misc import *
+from wic.utils.misc import exec_cmd, get_bitbake_var
+from wic.filemap import sparse_copy
-class RawCopyPlugin(SourcePlugin):
- name = 'rawcopy'
+logger = logging.getLogger('wic')
- @classmethod
- def do_install_disk(self, disk, disk_name, cr, workdir, oe_builddir,
- bootimg_dir, kernel_dir, native_sysroot):
- """
- Called after all partitions have been prepared and assembled into a
- disk image. Do nothing.
- """
- pass
+class RawCopyPlugin(SourcePlugin):
+ """
+ Populate partition content from raw image file.
+ """
- @classmethod
- def do_configure_partition(self, part, source_params, cr, cr_workdir,
- oe_builddir, bootimg_dir, kernel_dir,
- native_sysroot):
- """
- Called before do_prepare_partition(). Possibly prepare
- configuration files of some sort.
- """
- pass
+ name = 'rawcopy'
@classmethod
- def do_prepare_partition(self, part, source_params, cr, cr_workdir,
+ def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
oe_builddir, bootimg_dir, kernel_dir,
rootfs_dir, native_sysroot):
"""
Called to do the actual content population for a partition i.e. it
'prepares' the partition to be incorporated into the image.
"""
- if not bootimg_dir:
- bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
- if not bootimg_dir:
- msger.error("Couldn't find DEPLOY_DIR_IMAGE, exiting\n")
+ if not kernel_dir:
+ kernel_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+ if not kernel_dir:
+ raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
- msger.debug('Bootimg dir: %s' % bootimg_dir)
+ logger.debug('Kernel dir: %s', kernel_dir)
- if ('file' not in source_params):
- msger.error("No file specified\n")
- return
+ if 'file' not in source_params:
+ raise WicError("No file specified")
- src = os.path.join(bootimg_dir, source_params['file'])
- dst = src
+ src = os.path.join(kernel_dir, source_params['file'])
+ dst = os.path.join(cr_workdir, "%s.%s" % (source_params['file'], part.lineno))
- if ('skip' in source_params):
- dst = os.path.join(cr_workdir, source_params['file'])
- dd_cmd = "dd if=%s of=%s ibs=%s skip=1 conv=notrunc" % \
- (src, dst, source_params['skip'])
- exec_cmd(dd_cmd)
+ if 'skip' in source_params:
+ sparse_copy(src, dst, skip=int(source_params['skip']))
+ else:
+ sparse_copy(src, dst)
# get the size in the right units for kickstart (kB)
du_cmd = "du -Lbks %s" % dst
out = exec_cmd(du_cmd)
- filesize = out.split()[0]
+ filesize = int(out.split()[0])
if filesize > part.size:
part.size = filesize
part.source_file = dst
-
diff --git a/scripts/lib/wic/plugins/source/rootfs.py b/scripts/lib/wic/plugins/source/rootfs.py
index a432a18705..f2e2ca8a2b 100644
--- a/scripts/lib/wic/plugins/source/rootfs.py
+++ b/scripts/lib/wic/plugins/source/rootfs.py
@@ -25,21 +25,23 @@
# Joao Henrique Ferreira de Freitas <joaohf (at] gmail.com>
#
+import logging
import os
import shutil
-import re
-import tempfile
-
-from wic import kickstart, msger
-from wic.utils import misc, fs_related, errors, runner, cmdln
-from wic.conf import configmgr
-from wic.plugin import pluginmgr
-import wic.imager.direct as direct
+
+from oe.path import copyhardlinktree
+
+from wic import WicError
from wic.pluginbase import SourcePlugin
-from wic.utils.oe.misc import *
-from wic.imager.direct import DirectImageCreator
+from wic.utils.misc import get_bitbake_var, exec_cmd
+
+logger = logging.getLogger('wic')
class RootfsPlugin(SourcePlugin):
+ """
+ Populate partition content from a rootfs directory.
+ """
+
name = 'rootfs'
@staticmethod
@@ -47,22 +49,16 @@ class RootfsPlugin(SourcePlugin):
if os.path.isdir(rootfs_dir):
return rootfs_dir
- bitbake_env_lines = find_bitbake_env_lines(rootfs_dir)
- if not bitbake_env_lines:
- msg = "Couldn't get bitbake environment, exiting."
- msger.error(msg)
-
- image_rootfs_dir = find_artifact(bitbake_env_lines, "IMAGE_ROOTFS")
+ image_rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", rootfs_dir)
if not os.path.isdir(image_rootfs_dir):
- msg = "No valid artifact IMAGE_ROOTFS from image named"
- msg += " %s has been found at %s, exiting.\n" % \
- (rootfs_dir, image_rootfs_dir)
- msger.error(msg)
+ raise WicError("No valid artifact IMAGE_ROOTFS from image "
+ "named %s has been found at %s, exiting." %
+ (rootfs_dir, image_rootfs_dir))
return image_rootfs_dir
@classmethod
- def do_prepare_partition(self, part, source_params, cr, cr_workdir,
+ def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
oe_builddir, bootimg_dir, kernel_dir,
krootfs_dir, native_sysroot):
"""
@@ -70,23 +66,60 @@ class RootfsPlugin(SourcePlugin):
'prepares' the partition to be incorporated into the image.
In this case, prepare content for legacy bios boot partition.
"""
- if part.rootfs is None:
+ if part.rootfs_dir is None:
if not 'ROOTFS_DIR' in krootfs_dir:
- msg = "Couldn't find --rootfs-dir, exiting"
- msger.error(msg)
+ raise WicError("Couldn't find --rootfs-dir, exiting")
+
rootfs_dir = krootfs_dir['ROOTFS_DIR']
else:
- if part.rootfs in krootfs_dir:
- rootfs_dir = krootfs_dir[part.rootfs]
- elif part.rootfs:
- rootfs_dir = part.rootfs
+ if part.rootfs_dir in krootfs_dir:
+ rootfs_dir = krootfs_dir[part.rootfs_dir]
+ elif part.rootfs_dir:
+ rootfs_dir = part.rootfs_dir
else:
- msg = "Couldn't find --rootfs-dir=%s connection"
- msg += " or it is not a valid path, exiting"
- msger.error(msg % part.rootfs)
+ raise WicError("Couldn't find --rootfs-dir=%s connection or "
+ "it is not a valid path, exiting" % part.rootfs_dir)
+
+ real_rootfs_dir = cls.__get_rootfs_dir(rootfs_dir)
+
+ # Handle excluded paths.
+ if part.exclude_path is not None:
+ # We need a new rootfs directory we can delete files from. Copy to
+ # workdir.
+ new_rootfs = os.path.realpath(os.path.join(cr_workdir, "rootfs"))
+
+ if os.path.lexists(new_rootfs):
+ shutil.rmtree(os.path.join(new_rootfs))
+
+ copyhardlinktree(real_rootfs_dir, new_rootfs)
+
+ real_rootfs_dir = new_rootfs
+
+ for orig_path in part.exclude_path:
+ path = orig_path
+ if os.path.isabs(path):
+ msger.error("Must be relative: --exclude-path=%s" % orig_path)
+
+ full_path = os.path.realpath(os.path.join(new_rootfs, path))
- real_rootfs_dir = self.__get_rootfs_dir(rootfs_dir)
+ # Disallow climbing outside of parent directory using '..',
+ # because doing so could be quite disastrous (we will delete the
+ # directory).
+ if not full_path.startswith(new_rootfs):
+ msger.error("'%s' points to a path outside the rootfs" % orig_path)
- part.set_rootfs(real_rootfs_dir)
- part.prepare_rootfs(cr_workdir, oe_builddir, real_rootfs_dir, native_sysroot)
+ if path.endswith(os.sep):
+ # Delete content only.
+ for entry in os.listdir(full_path):
+ full_entry = os.path.join(full_path, entry)
+ if os.path.isdir(full_entry) and not os.path.islink(full_entry):
+ shutil.rmtree(full_entry)
+ else:
+ os.remove(full_entry)
+ else:
+ # Delete whole directory.
+ shutil.rmtree(full_path)
+ part.rootfs_dir = real_rootfs_dir
+ part.prepare_rootfs(cr_workdir, oe_builddir,
+ real_rootfs_dir, native_sysroot)
diff --git a/scripts/lib/wic/test b/scripts/lib/wic/test
deleted file mode 100644
index 9daeafb986..0000000000
--- a/scripts/lib/wic/test
+++ /dev/null
@@ -1 +0,0 @@
-test
diff --git a/scripts/lib/wic/utils/cmdln.py b/scripts/lib/wic/utils/cmdln.py
deleted file mode 100644
index b099473ee4..0000000000
--- a/scripts/lib/wic/utils/cmdln.py
+++ /dev/null
@@ -1,1586 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2002-2007 ActiveState Software Inc.
-# License: MIT (see LICENSE.txt for license details)
-# Author: Trent Mick
-# Home: http://trentm.com/projects/cmdln/
-
-"""An improvement on Python's standard cmd.py module.
-
-As with cmd.py, this module provides "a simple framework for writing
-line-oriented command intepreters." This module provides a 'RawCmdln'
-class that fixes some design flaws in cmd.Cmd, making it more scalable
-and nicer to use for good 'cvs'- or 'svn'-style command line interfaces
-or simple shells. And it provides a 'Cmdln' class that add
-optparse-based option processing. Basically you use it like this:
-
- import cmdln
-
- class MySVN(cmdln.Cmdln):
- name = "svn"
-
- @cmdln.alias('stat', 'st')
- @cmdln.option('-v', '--verbose', action='store_true'
- help='print verbose information')
- def do_status(self, subcmd, opts, *paths):
- print "handle 'svn status' command"
-
- #...
-
- if __name__ == "__main__":
- shell = MySVN()
- retval = shell.main()
- sys.exit(retval)
-
-See the README.txt or <http://trentm.com/projects/cmdln/> for more
-details.
-"""
-
-__version_info__ = (1, 1, 2)
-__version__ = '.'.join(map(str, __version_info__))
-
-import os
-import sys
-import re
-import cmd
-import optparse
-from pprint import pprint
-import sys
-
-
-
-
-#---- globals
-
-LOOP_ALWAYS, LOOP_NEVER, LOOP_IF_EMPTY = range(3)
-
-# An unspecified optional argument when None is a meaningful value.
-_NOT_SPECIFIED = ("Not", "Specified")
-
-# Pattern to match a TypeError message from a call that
-# failed because of incorrect number of arguments (see
-# Python/getargs.c).
-_INCORRECT_NUM_ARGS_RE = re.compile(
- r"(takes [\w ]+ )(\d+)( arguments? \()(\d+)( given\))")
-
-
-
-#---- exceptions
-
-class CmdlnError(Exception):
- """A cmdln.py usage error."""
- def __init__(self, msg):
- self.msg = msg
- def __str__(self):
- return self.msg
-
-class CmdlnUserError(Exception):
- """An error by a user of a cmdln-based tool/shell."""
- pass
-
-
-
-#---- public methods and classes
-
-def alias(*aliases):
- """Decorator to add aliases for Cmdln.do_* command handlers.
-
- Example:
- class MyShell(cmdln.Cmdln):
- @cmdln.alias("!", "sh")
- def do_shell(self, argv):
- #...implement 'shell' command
- """
- def decorate(f):
- if not hasattr(f, "aliases"):
- f.aliases = []
- f.aliases += aliases
- return f
- return decorate
-
-
-class RawCmdln(cmd.Cmd):
- """An improved (on cmd.Cmd) framework for building multi-subcommand
- scripts (think "svn" & "cvs") and simple shells (think "pdb" and
- "gdb").
-
- A simple example:
-
- import cmdln
-
- class MySVN(cmdln.RawCmdln):
- name = "svn"
-
- @cmdln.aliases('stat', 'st')
- def do_status(self, argv):
- print "handle 'svn status' command"
-
- if __name__ == "__main__":
- shell = MySVN()
- retval = shell.main()
- sys.exit(retval)
-
- See <http://trentm.com/projects/cmdln> for more information.
- """
- name = None # if unset, defaults basename(sys.argv[0])
- prompt = None # if unset, defaults to self.name+"> "
- version = None # if set, default top-level options include --version
-
- # Default messages for some 'help' command error cases.
- # They are interpolated with one arg: the command.
- nohelp = "no help on '%s'"
- unknowncmd = "unknown command: '%s'"
-
- helpindent = '' # string with which to indent help output
-
- def __init__(self, completekey='tab',
- stdin=None, stdout=None, stderr=None):
- """Cmdln(completekey='tab', stdin=None, stdout=None, stderr=None)
-
- The optional argument 'completekey' is the readline name of a
- completion key; it defaults to the Tab key. If completekey is
- not None and the readline module is available, command completion
- is done automatically.
-
- The optional arguments 'stdin', 'stdout' and 'stderr' specify
- alternate input, output and error output file objects; if not
- specified, sys.* are used.
-
- If 'stdout' but not 'stderr' is specified, stdout is used for
- error output. This is to provide least surprise for users used
- to only the 'stdin' and 'stdout' options with cmd.Cmd.
- """
- import sys
- if self.name is None:
- self.name = os.path.basename(sys.argv[0])
- if self.prompt is None:
- self.prompt = self.name+"> "
- self._name_str = self._str(self.name)
- self._prompt_str = self._str(self.prompt)
- if stdin is not None:
- self.stdin = stdin
- else:
- self.stdin = sys.stdin
- if stdout is not None:
- self.stdout = stdout
- else:
- self.stdout = sys.stdout
- if stderr is not None:
- self.stderr = stderr
- elif stdout is not None:
- self.stderr = stdout
- else:
- self.stderr = sys.stderr
- self.cmdqueue = []
- self.completekey = completekey
- self.cmdlooping = False
-
- def get_optparser(self):
- """Hook for subclasses to set the option parser for the
- top-level command/shell.
-
- This option parser is used retrieved and used by `.main()' to
- handle top-level options.
-
- The default implements a single '-h|--help' option. Sub-classes
- can return None to have no options at the top-level. Typically
- an instance of CmdlnOptionParser should be returned.
- """
- version = (self.version is not None
- and "%s %s" % (self._name_str, self.version)
- or None)
- return CmdlnOptionParser(self, version=version)
-
- def postoptparse(self):
- """Hook method executed just after `.main()' parses top-level
- options.
-
- When called `self.options' holds the results of the option parse.
- """
- pass
-
- def main(self, argv=None, loop=LOOP_NEVER):
- """A possible mainline handler for a script, like so:
-
- import cmdln
- class MyCmd(cmdln.Cmdln):
- name = "mycmd"
- ...
-
- if __name__ == "__main__":
- MyCmd().main()
-
- By default this will use sys.argv to issue a single command to
- 'MyCmd', then exit. The 'loop' argument can be use to control
- interactive shell behaviour.
-
- Arguments:
- "argv" (optional, default sys.argv) is the command to run.
- It must be a sequence, where the first element is the
- command name and subsequent elements the args for that
- command.
- "loop" (optional, default LOOP_NEVER) is a constant
- indicating if a command loop should be started (i.e. an
- interactive shell). Valid values (constants on this module):
- LOOP_ALWAYS start loop and run "argv", if any
- LOOP_NEVER run "argv" (or .emptyline()) and exit
- LOOP_IF_EMPTY run "argv", if given, and exit;
- otherwise, start loop
- """
- if argv is None:
- import sys
- argv = sys.argv
- else:
- argv = argv[:] # don't modify caller's list
-
- self.optparser = self.get_optparser()
- if self.optparser: # i.e. optparser=None means don't process for opts
- try:
- self.options, args = self.optparser.parse_args(argv[1:])
- except CmdlnUserError, ex:
- msg = "%s: %s\nTry '%s help' for info.\n"\
- % (self.name, ex, self.name)
- self.stderr.write(self._str(msg))
- self.stderr.flush()
- return 1
- except StopOptionProcessing, ex:
- return 0
- else:
- self.options, args = None, argv[1:]
- self.postoptparse()
-
- if loop == LOOP_ALWAYS:
- if args:
- self.cmdqueue.append(args)
- return self.cmdloop()
- elif loop == LOOP_NEVER:
- if args:
- return self.cmd(args)
- else:
- return self.emptyline()
- elif loop == LOOP_IF_EMPTY:
- if args:
- return self.cmd(args)
- else:
- return self.cmdloop()
-
- def cmd(self, argv):
- """Run one command and exit.
-
- "argv" is the arglist for the command to run. argv[0] is the
- command to run. If argv is an empty list then the
- 'emptyline' handler is run.
-
- Returns the return value from the command handler.
- """
- assert isinstance(argv, (list, tuple)), \
- "'argv' is not a sequence: %r" % argv
- retval = None
- try:
- argv = self.precmd(argv)
- retval = self.onecmd(argv)
- self.postcmd(argv)
- except:
- if not self.cmdexc(argv):
- raise
- retval = 1
- return retval
-
- def _str(self, s):
- """Safely convert the given str/unicode to a string for printing."""
- try:
- return str(s)
- except UnicodeError:
- #XXX What is the proper encoding to use here? 'utf-8' seems
- # to work better than "getdefaultencoding" (usually
- # 'ascii'), on OS X at least.
- #import sys
- #return s.encode(sys.getdefaultencoding(), "replace")
- return s.encode("utf-8", "replace")
-
- def cmdloop(self, intro=None):
- """Repeatedly issue a prompt, accept input, parse into an argv, and
- dispatch (via .precmd(), .onecmd() and .postcmd()), passing them
- the argv. In other words, start a shell.
-
- "intro" (optional) is a introductory message to print when
- starting the command loop. This overrides the class
- "intro" attribute, if any.
- """
- self.cmdlooping = True
- self.preloop()
- if self.use_rawinput and self.completekey:
- try:
- import readline
- self.old_completer = readline.get_completer()
- readline.set_completer(self.complete)
- readline.parse_and_bind(self.completekey+": complete")
- except ImportError:
- pass
- try:
- if intro is None:
- intro = self.intro
- if intro:
- intro_str = self._str(intro)
- self.stdout.write(intro_str+'\n')
- self.stop = False
- retval = None
- while not self.stop:
- if self.cmdqueue:
- argv = self.cmdqueue.pop(0)
- assert isinstance(argv, (list, tuple)), \
- "item on 'cmdqueue' is not a sequence: %r" % argv
- else:
- if self.use_rawinput:
- try:
- line = raw_input(self._prompt_str)
- except EOFError:
- line = 'EOF'
- else:
- self.stdout.write(self._prompt_str)
- self.stdout.flush()
- line = self.stdin.readline()
- if not len(line):
- line = 'EOF'
- else:
- line = line[:-1] # chop '\n'
- argv = line2argv(line)
- try:
- argv = self.precmd(argv)
- retval = self.onecmd(argv)
- self.postcmd(argv)
- except:
- if not self.cmdexc(argv):
- raise
- retval = 1
- self.lastretval = retval
- self.postloop()
- finally:
- if self.use_rawinput and self.completekey:
- try:
- import readline
- readline.set_completer(self.old_completer)
- except ImportError:
- pass
- self.cmdlooping = False
- return retval
-
- def precmd(self, argv):
- """Hook method executed just before the command argv is
- interpreted, but after the input prompt is generated and issued.
-
- "argv" is the cmd to run.
-
- Returns an argv to run (i.e. this method can modify the command
- to run).
- """
- return argv
-
- def postcmd(self, argv):
- """Hook method executed just after a command dispatch is finished.
-
- "argv" is the command that was run.
- """
- pass
-
- def cmdexc(self, argv):
- """Called if an exception is raised in any of precmd(), onecmd(),
- or postcmd(). If True is returned, the exception is deemed to have
- been dealt with. Otherwise, the exception is re-raised.
-
- The default implementation handles CmdlnUserError's, which
- typically correspond to user error in calling commands (as
- opposed to programmer error in the design of the script using
- cmdln.py).
- """
- import sys
- type, exc, traceback = sys.exc_info()
- if isinstance(exc, CmdlnUserError):
- msg = "%s %s: %s\nTry '%s help %s' for info.\n"\
- % (self.name, argv[0], exc, self.name, argv[0])
- self.stderr.write(self._str(msg))
- self.stderr.flush()
- return True
-
- def onecmd(self, argv):
- if not argv:
- return self.emptyline()
- self.lastcmd = argv
- cmdname = self._get_canonical_cmd_name(argv[0])
- if cmdname:
- handler = self._get_cmd_handler(cmdname)
- if handler:
- return self._dispatch_cmd(handler, argv)
- return self.default(argv)
-
- def _dispatch_cmd(self, handler, argv):
- return handler(argv)
-
- def default(self, argv):
- """Hook called to handle a command for which there is no handler.
-
- "argv" is the command and arguments to run.
-
- The default implementation writes and error message to stderr
- and returns an error exit status.
-
- Returns a numeric command exit status.
- """
- errmsg = self._str(self.unknowncmd % (argv[0],))
- if self.cmdlooping:
- self.stderr.write(errmsg+"\n")
- else:
- self.stderr.write("%s: %s\nTry '%s help' for info.\n"
- % (self._name_str, errmsg, self._name_str))
- self.stderr.flush()
- return 1
-
- def parseline(self, line):
- # This is used by Cmd.complete (readline completer function) to
- # massage the current line buffer before completion processing.
- # We override to drop special '!' handling.
- line = line.strip()
- if not line:
- return None, None, line
- elif line[0] == '?':
- line = 'help ' + line[1:]
- i, n = 0, len(line)
- while i < n and line[i] in self.identchars: i = i+1
- cmd, arg = line[:i], line[i:].strip()
- return cmd, arg, line
-
- def helpdefault(self, cmd, known):
- """Hook called to handle help on a command for which there is no
- help handler.
-
- "cmd" is the command name on which help was requested.
- "known" is a boolean indicating if this command is known
- (i.e. if there is a handler for it).
-
- Returns a return code.
- """
- if known:
- msg = self._str(self.nohelp % (cmd,))
- if self.cmdlooping:
- self.stderr.write(msg + '\n')
- else:
- self.stderr.write("%s: %s\n" % (self.name, msg))
- else:
- msg = self.unknowncmd % (cmd,)
- if self.cmdlooping:
- self.stderr.write(msg + '\n')
- else:
- self.stderr.write("%s: %s\n"
- "Try '%s help' for info.\n"
- % (self.name, msg, self.name))
- self.stderr.flush()
- return 1
-
- def do_help(self, argv):
- """${cmd_name}: give detailed help on a specific sub-command
-
- Usage:
- ${name} help [COMMAND]
- """
- if len(argv) > 1: # asking for help on a particular command
- doc = None
- cmdname = self._get_canonical_cmd_name(argv[1]) or argv[1]
- if not cmdname:
- return self.helpdefault(argv[1], False)
- else:
- helpfunc = getattr(self, "help_"+cmdname, None)
- if helpfunc:
- doc = helpfunc()
- else:
- handler = self._get_cmd_handler(cmdname)
- if handler:
- doc = handler.__doc__
- if doc is None:
- return self.helpdefault(argv[1], handler != None)
- else: # bare "help" command
- doc = self.__class__.__doc__ # try class docstring
- if doc is None:
- # Try to provide some reasonable useful default help.
- if self.cmdlooping: prefix = ""
- else: prefix = self.name+' '
- doc = """Usage:
- %sCOMMAND [ARGS...]
- %shelp [COMMAND]
-
- ${option_list}
- ${command_list}
- ${help_list}
- """ % (prefix, prefix)
- cmdname = None
-
- if doc: # *do* have help content, massage and print that
- doc = self._help_reindent(doc)
- doc = self._help_preprocess(doc, cmdname)
- doc = doc.rstrip() + '\n' # trim down trailing space
- self.stdout.write(self._str(doc))
- self.stdout.flush()
- do_help.aliases = ["?"]
-
- def _help_reindent(self, help, indent=None):
- """Hook to re-indent help strings before writing to stdout.
-
- "help" is the help content to re-indent
- "indent" is a string with which to indent each line of the
- help content after normalizing. If unspecified or None
- then the default is use: the 'self.helpindent' class
- attribute. By default this is the empty string, i.e.
- no indentation.
-
- By default, all common leading whitespace is removed and then
- the lot is indented by 'self.helpindent'. When calculating the
- common leading whitespace the first line is ignored -- hence
- help content for Conan can be written as follows and have the
- expected indentation:
-
- def do_crush(self, ...):
- '''${cmd_name}: crush your enemies, see them driven before you...
-
- c.f. Conan the Barbarian'''
- """
- if indent is None:
- indent = self.helpindent
- lines = help.splitlines(0)
- _dedentlines(lines, skip_first_line=True)
- lines = [(indent+line).rstrip() for line in lines]
- return '\n'.join(lines)
-
- def _help_preprocess(self, help, cmdname):
- """Hook to preprocess a help string before writing to stdout.
-
- "help" is the help string to process.
- "cmdname" is the canonical sub-command name for which help
- is being given, or None if the help is not specific to a
- command.
-
- By default the following template variables are interpolated in
- help content. (Note: these are similar to Python 2.4's
- string.Template interpolation but not quite.)
-
- ${name}
- The tool's/shell's name, i.e. 'self.name'.
- ${option_list}
- A formatted table of options for this shell/tool.
- ${command_list}
- A formatted table of available sub-commands.
- ${help_list}
- A formatted table of additional help topics (i.e. 'help_*'
- methods with no matching 'do_*' method).
- ${cmd_name}
- The name (and aliases) for this sub-command formatted as:
- "NAME (ALIAS1, ALIAS2, ...)".
- ${cmd_usage}
- A formatted usage block inferred from the command function
- signature.
- ${cmd_option_list}
- A formatted table of options for this sub-command. (This is
- only available for commands using the optparse integration,
- i.e. using @cmdln.option decorators or manually setting the
- 'optparser' attribute on the 'do_*' method.)
-
- Returns the processed help.
- """
- preprocessors = {
- "${name}": self._help_preprocess_name,
- "${option_list}": self._help_preprocess_option_list,
- "${command_list}": self._help_preprocess_command_list,
- "${help_list}": self._help_preprocess_help_list,
- "${cmd_name}": self._help_preprocess_cmd_name,
- "${cmd_usage}": self._help_preprocess_cmd_usage,
- "${cmd_option_list}": self._help_preprocess_cmd_option_list,
- }
-
- for marker, preprocessor in preprocessors.items():
- if marker in help:
- help = preprocessor(help, cmdname)
- return help
-
- def _help_preprocess_name(self, help, cmdname=None):
- return help.replace("${name}", self.name)
-
- def _help_preprocess_option_list(self, help, cmdname=None):
- marker = "${option_list}"
- indent, indent_width = _get_indent(marker, help)
- suffix = _get_trailing_whitespace(marker, help)
-
- if self.optparser:
- # Setup formatting options and format.
- # - Indentation of 4 is better than optparse default of 2.
- # C.f. Damian Conway's discussion of this in Perl Best
- # Practices.
- self.optparser.formatter.indent_increment = 4
- self.optparser.formatter.current_indent = indent_width
- block = self.optparser.format_option_help() + '\n'
- else:
- block = ""
-
- help = help.replace(indent+marker+suffix, block, 1)
- return help
-
-
- def _help_preprocess_command_list(self, help, cmdname=None):
- marker = "${command_list}"
- indent, indent_width = _get_indent(marker, help)
- suffix = _get_trailing_whitespace(marker, help)
-
- # Find any aliases for commands.
- token2canonical = self._get_canonical_map()
- aliases = {}
- for token, cmdname in token2canonical.items():
- if token == cmdname: continue
- aliases.setdefault(cmdname, []).append(token)
-
- # Get the list of (non-hidden) commands and their
- # documentation, if any.
- cmdnames = {} # use a dict to strip duplicates
- for attr in self.get_names():
- if attr.startswith("do_"):
- cmdnames[attr[3:]] = True
- cmdnames = cmdnames.keys()
- cmdnames.sort()
- linedata = []
- for cmdname in cmdnames:
- if aliases.get(cmdname):
- a = aliases[cmdname]
- a.sort()
- cmdstr = "%s (%s)" % (cmdname, ", ".join(a))
- else:
- cmdstr = cmdname
- doc = None
- try:
- helpfunc = getattr(self, 'help_'+cmdname)
- except AttributeError:
- handler = self._get_cmd_handler(cmdname)
- if handler:
- doc = handler.__doc__
- else:
- doc = helpfunc()
-
- # Strip "${cmd_name}: " from the start of a command's doc. Best
- # practice dictates that command help strings begin with this, but
- # it isn't at all wanted for the command list.
- to_strip = "${cmd_name}:"
- if doc and doc.startswith(to_strip):
- #log.debug("stripping %r from start of %s's help string",
- # to_strip, cmdname)
- doc = doc[len(to_strip):].lstrip()
- linedata.append( (cmdstr, doc) )
-
- if linedata:
- subindent = indent + ' '*4
- lines = _format_linedata(linedata, subindent, indent_width+4)
- block = indent + "Commands:\n" \
- + '\n'.join(lines) + "\n\n"
- help = help.replace(indent+marker+suffix, block, 1)
- return help
-
- def _gen_names_and_attrs(self):
- # Inheritance says we have to look in class and
- # base classes; order is not important.
- names = []
- classes = [self.__class__]
- while classes:
- aclass = classes.pop(0)
- if aclass.__bases__:
- classes = classes + list(aclass.__bases__)
- for name in dir(aclass):
- yield (name, getattr(aclass, name))
-
- def _help_preprocess_help_list(self, help, cmdname=None):
- marker = "${help_list}"
- indent, indent_width = _get_indent(marker, help)
- suffix = _get_trailing_whitespace(marker, help)
-
- # Determine the additional help topics, if any.
- helpnames = {}
- token2cmdname = self._get_canonical_map()
- for attrname, attr in self._gen_names_and_attrs():
- if not attrname.startswith("help_"): continue
- helpname = attrname[5:]
- if helpname not in token2cmdname:
- helpnames[helpname] = attr
-
- if helpnames:
- linedata = [(n, a.__doc__ or "") for n, a in helpnames.items()]
- linedata.sort()
-
- subindent = indent + ' '*4
- lines = _format_linedata(linedata, subindent, indent_width+4)
- block = (indent
- + "Additional help topics (run `%s help TOPIC'):\n" % self.name
- + '\n'.join(lines)
- + "\n\n")
- else:
- block = ''
- help = help.replace(indent+marker+suffix, block, 1)
- return help
-
- def _help_preprocess_cmd_name(self, help, cmdname=None):
- marker = "${cmd_name}"
- handler = self._get_cmd_handler(cmdname)
- if not handler:
- raise CmdlnError("cannot preprocess '%s' into help string: "
- "could not find command handler for %r"
- % (marker, cmdname))
- s = cmdname
- if hasattr(handler, "aliases"):
- s += " (%s)" % (", ".join(handler.aliases))
- help = help.replace(marker, s)
- return help
-
- #TODO: this only makes sense as part of the Cmdln class.
- # Add hooks to add help preprocessing template vars and put
- # this one on that class.
- def _help_preprocess_cmd_usage(self, help, cmdname=None):
- marker = "${cmd_usage}"
- handler = self._get_cmd_handler(cmdname)
- if not handler:
- raise CmdlnError("cannot preprocess '%s' into help string: "
- "could not find command handler for %r"
- % (marker, cmdname))
- indent, indent_width = _get_indent(marker, help)
- suffix = _get_trailing_whitespace(marker, help)
-
- # Extract the introspection bits we need.
- func = handler.im_func
- if func.func_defaults:
- func_defaults = list(func.func_defaults)
- else:
- func_defaults = []
- co_argcount = func.func_code.co_argcount
- co_varnames = func.func_code.co_varnames
- co_flags = func.func_code.co_flags
- CO_FLAGS_ARGS = 4
- CO_FLAGS_KWARGS = 8
-
- # Adjust argcount for possible *args and **kwargs arguments.
- argcount = co_argcount
- if co_flags & CO_FLAGS_ARGS: argcount += 1
- if co_flags & CO_FLAGS_KWARGS: argcount += 1
-
- # Determine the usage string.
- usage = "%s %s" % (self.name, cmdname)
- if argcount <= 2: # handler ::= do_FOO(self, argv)
- usage += " [ARGS...]"
- elif argcount >= 3: # handler ::= do_FOO(self, subcmd, opts, ...)
- argnames = list(co_varnames[3:argcount])
- tail = ""
- if co_flags & CO_FLAGS_KWARGS:
- name = argnames.pop(-1)
- import warnings
- # There is no generally accepted mechanism for passing
- # keyword arguments from the command line. Could
- # *perhaps* consider: arg=value arg2=value2 ...
- warnings.warn("argument '**%s' on '%s.%s' command "
- "handler will never get values"
- % (name, self.__class__.__name__,
- func.func_name))
- if co_flags & CO_FLAGS_ARGS:
- name = argnames.pop(-1)
- tail = "[%s...]" % name.upper()
- while func_defaults:
- func_defaults.pop(-1)
- name = argnames.pop(-1)
- tail = "[%s%s%s]" % (name.upper(), (tail and ' ' or ''), tail)
- while argnames:
- name = argnames.pop(-1)
- tail = "%s %s" % (name.upper(), tail)
- usage += ' ' + tail
-
- block_lines = [
- self.helpindent + "Usage:",
- self.helpindent + ' '*4 + usage
- ]
- block = '\n'.join(block_lines) + '\n\n'
-
- help = help.replace(indent+marker+suffix, block, 1)
- return help
-
- #TODO: this only makes sense as part of the Cmdln class.
- # Add hooks to add help preprocessing template vars and put
- # this one on that class.
- def _help_preprocess_cmd_option_list(self, help, cmdname=None):
- marker = "${cmd_option_list}"
- handler = self._get_cmd_handler(cmdname)
- if not handler:
- raise CmdlnError("cannot preprocess '%s' into help string: "
- "could not find command handler for %r"
- % (marker, cmdname))
- indent, indent_width = _get_indent(marker, help)
- suffix = _get_trailing_whitespace(marker, help)
- if hasattr(handler, "optparser"):
- # Setup formatting options and format.
- # - Indentation of 4 is better than optparse default of 2.
- # C.f. Damian Conway's discussion of this in Perl Best
- # Practices.
- handler.optparser.formatter.indent_increment = 4
- handler.optparser.formatter.current_indent = indent_width
- block = handler.optparser.format_option_help() + '\n'
- else:
- block = ""
-
- help = help.replace(indent+marker+suffix, block, 1)
- return help
-
- def _get_canonical_cmd_name(self, token):
- map = self._get_canonical_map()
- return map.get(token, None)
-
- def _get_canonical_map(self):
- """Return a mapping of available command names and aliases to
- their canonical command name.
- """
- cacheattr = "_token2canonical"
- if not hasattr(self, cacheattr):
- # Get the list of commands and their aliases, if any.
- token2canonical = {}
- cmd2funcname = {} # use a dict to strip duplicates
- for attr in self.get_names():
- if attr.startswith("do_"): cmdname = attr[3:]
- elif attr.startswith("_do_"): cmdname = attr[4:]
- else:
- continue
- cmd2funcname[cmdname] = attr
- token2canonical[cmdname] = cmdname
- for cmdname, funcname in cmd2funcname.items(): # add aliases
- func = getattr(self, funcname)
- aliases = getattr(func, "aliases", [])
- for alias in aliases:
- if alias in cmd2funcname:
- import warnings
- warnings.warn("'%s' alias for '%s' command conflicts "
- "with '%s' handler"
- % (alias, cmdname, cmd2funcname[alias]))
- continue
- token2canonical[alias] = cmdname
- setattr(self, cacheattr, token2canonical)
- return getattr(self, cacheattr)
-
- def _get_cmd_handler(self, cmdname):
- handler = None
- try:
- handler = getattr(self, 'do_' + cmdname)
- except AttributeError:
- try:
- # Private command handlers begin with "_do_".
- handler = getattr(self, '_do_' + cmdname)
- except AttributeError:
- pass
- return handler
-
- def _do_EOF(self, argv):
- # Default EOF handler
- # Note: an actual EOF is redirected to this command.
- #TODO: separate name for this. Currently it is available from
- # command-line. Is that okay?
- self.stdout.write('\n')
- self.stdout.flush()
- self.stop = True
-
- def emptyline(self):
- # Different from cmd.Cmd: don't repeat the last command for an
- # emptyline.
- if self.cmdlooping:
- pass
- else:
- return self.do_help(["help"])
-
-
-#---- optparse.py extension to fix (IMO) some deficiencies
-#
-# See the class _OptionParserEx docstring for details.
-#
-
-class StopOptionProcessing(Exception):
- """Indicate that option *and argument* processing should stop
- cleanly. This is not an error condition. It is similar in spirit to
- StopIteration. This is raised by _OptionParserEx's default "help"
- and "version" option actions and can be raised by custom option
- callbacks too.
-
- Hence the typical CmdlnOptionParser (a subclass of _OptionParserEx)
- usage is:
-
- parser = CmdlnOptionParser(mycmd)
- parser.add_option("-f", "--force", dest="force")
- ...
- try:
- opts, args = parser.parse_args()
- except StopOptionProcessing:
- # normal termination, "--help" was probably given
- sys.exit(0)
- """
-
-class _OptionParserEx(optparse.OptionParser):
- """An optparse.OptionParser that uses exceptions instead of sys.exit.
-
- This class is an extension of optparse.OptionParser that differs
- as follows:
- - Correct (IMO) the default OptionParser error handling to never
- sys.exit(). Instead OptParseError exceptions are passed through.
- - Add the StopOptionProcessing exception (a la StopIteration) to
- indicate normal termination of option processing.
- See StopOptionProcessing's docstring for details.
-
- I'd also like to see the following in the core optparse.py, perhaps
- as a RawOptionParser which would serve as a base class for the more
- generally used OptionParser (that works as current):
- - Remove the implicit addition of the -h|--help and --version
- options. They can get in the way (e.g. if want '-?' and '-V' for
- these as well) and it is not hard to do:
- optparser.add_option("-h", "--help", action="help")
- optparser.add_option("--version", action="version")
- These are good practices, just not valid defaults if they can
- get in the way.
- """
- def error(self, msg):
- raise optparse.OptParseError(msg)
-
- def exit(self, status=0, msg=None):
- if status == 0:
- raise StopOptionProcessing(msg)
- else:
- #TODO: don't lose status info here
- raise optparse.OptParseError(msg)
-
-
-
-#---- optparse.py-based option processing support
-
-class CmdlnOptionParser(_OptionParserEx):
- """An optparse.OptionParser class more appropriate for top-level
- Cmdln options. For parsing of sub-command options, see
- SubCmdOptionParser.
-
- Changes:
- - disable_interspersed_args() by default, because a Cmdln instance
- has sub-commands which may themselves have options.
- - Redirect print_help() to the Cmdln.do_help() which is better
- equiped to handle the "help" action.
- - error() will raise a CmdlnUserError: OptionParse.error() is meant
- to be called for user errors. Raising a well-known error here can
- make error handling clearer.
- - Also see the changes in _OptionParserEx.
- """
- def __init__(self, cmdln, **kwargs):
- self.cmdln = cmdln
- kwargs["prog"] = self.cmdln.name
- _OptionParserEx.__init__(self, **kwargs)
- self.disable_interspersed_args()
-
- def print_help(self, file=None):
- self.cmdln.onecmd(["help"])
-
- def error(self, msg):
- raise CmdlnUserError(msg)
-
-
-class SubCmdOptionParser(_OptionParserEx):
- def set_cmdln_info(self, cmdln, subcmd):
- """Called by Cmdln to pass relevant info about itself needed
- for print_help().
- """
- self.cmdln = cmdln
- self.subcmd = subcmd
-
- def print_help(self, file=None):
- self.cmdln.onecmd(["help", self.subcmd])
-
- def error(self, msg):
- raise CmdlnUserError(msg)
-
-
-def option(*args, **kwargs):
- """Decorator to add an option to the optparser argument of a Cmdln
- subcommand.
-
- Example:
- class MyShell(cmdln.Cmdln):
- @cmdln.option("-f", "--force", help="force removal")
- def do_remove(self, subcmd, opts, *args):
- #...
- """
- #XXX Is there a possible optimization for many options to not have a
- # large stack depth here?
- def decorate(f):
- if not hasattr(f, "optparser"):
- f.optparser = SubCmdOptionParser()
- f.optparser.add_option(*args, **kwargs)
- return f
- return decorate
-
-
-class Cmdln(RawCmdln):
- """An improved (on cmd.Cmd) framework for building multi-subcommand
- scripts (think "svn" & "cvs") and simple shells (think "pdb" and
- "gdb").
-
- A simple example:
-
- import cmdln
-
- class MySVN(cmdln.Cmdln):
- name = "svn"
-
- @cmdln.aliases('stat', 'st')
- @cmdln.option('-v', '--verbose', action='store_true'
- help='print verbose information')
- def do_status(self, subcmd, opts, *paths):
- print "handle 'svn status' command"
-
- #...
-
- if __name__ == "__main__":
- shell = MySVN()
- retval = shell.main()
- sys.exit(retval)
-
- 'Cmdln' extends 'RawCmdln' by providing optparse option processing
- integration. See this class' _dispatch_cmd() docstring and
- <http://trentm.com/projects/cmdln> for more information.
- """
- def _dispatch_cmd(self, handler, argv):
- """Introspect sub-command handler signature to determine how to
- dispatch the command. The raw handler provided by the base
- 'RawCmdln' class is still supported:
-
- def do_foo(self, argv):
- # 'argv' is the vector of command line args, argv[0] is
- # the command name itself (i.e. "foo" or an alias)
- pass
-
- In addition, if the handler has more than 2 arguments option
- processing is automatically done (using optparse):
-
- @cmdln.option('-v', '--verbose', action='store_true')
- def do_bar(self, subcmd, opts, *args):
- # subcmd = <"bar" or an alias>
- # opts = <an optparse.Values instance>
- if opts.verbose:
- print "lots of debugging output..."
- # args = <tuple of arguments>
- for arg in args:
- bar(arg)
-
- TODO: explain that "*args" can be other signatures as well.
-
- The `cmdln.option` decorator corresponds to an `add_option()`
- method call on an `optparse.OptionParser` instance.
-
- You can declare a specific number of arguments:
-
- @cmdln.option('-v', '--verbose', action='store_true')
- def do_bar2(self, subcmd, opts, bar_one, bar_two):
- #...
-
- and an appropriate error message will be raised/printed if the
- command is called with a different number of args.
- """
- co_argcount = handler.im_func.func_code.co_argcount
- if co_argcount == 2: # handler ::= do_foo(self, argv)
- return handler(argv)
- elif co_argcount >= 3: # handler ::= do_foo(self, subcmd, opts, ...)
- try:
- optparser = handler.optparser
- except AttributeError:
- optparser = handler.im_func.optparser = SubCmdOptionParser()
- assert isinstance(optparser, SubCmdOptionParser)
- optparser.set_cmdln_info(self, argv[0])
- try:
- opts, args = optparser.parse_args(argv[1:])
- except StopOptionProcessing:
- #TODO: this doesn't really fly for a replacement of
- # optparse.py behaviour, does it?
- return 0 # Normal command termination
-
- try:
- return handler(argv[0], opts, *args)
- except TypeError, ex:
- # Some TypeError's are user errors:
- # do_foo() takes at least 4 arguments (3 given)
- # do_foo() takes at most 5 arguments (6 given)
- # do_foo() takes exactly 5 arguments (6 given)
- # Raise CmdlnUserError for these with a suitably
- # massaged error message.
- import sys
- tb = sys.exc_info()[2] # the traceback object
- if tb.tb_next is not None:
- # If the traceback is more than one level deep, then the
- # TypeError do *not* happen on the "handler(...)" call
- # above. In that we don't want to handle it specially
- # here: it would falsely mask deeper code errors.
- raise
- msg = ex.args[0]
- match = _INCORRECT_NUM_ARGS_RE.search(msg)
- if match:
- msg = list(match.groups())
- msg[1] = int(msg[1]) - 3
- if msg[1] == 1:
- msg[2] = msg[2].replace("arguments", "argument")
- msg[3] = int(msg[3]) - 3
- msg = ''.join(map(str, msg))
- raise CmdlnUserError(msg)
- else:
- raise
- else:
- raise CmdlnError("incorrect argcount for %s(): takes %d, must "
- "take 2 for 'argv' signature or 3+ for 'opts' "
- "signature" % (handler.__name__, co_argcount))
-
-
-
-#---- internal support functions
-
-def _format_linedata(linedata, indent, indent_width):
- """Format specific linedata into a pleasant layout.
-
- "linedata" is a list of 2-tuples of the form:
- (<item-display-string>, <item-docstring>)
- "indent" is a string to use for one level of indentation
- "indent_width" is a number of columns by which the
- formatted data will be indented when printed.
-
- The <item-display-string> column is held to 15 columns.
- """
- lines = []
- WIDTH = 78 - indent_width
- SPACING = 2
- NAME_WIDTH_LOWER_BOUND = 13
- NAME_WIDTH_UPPER_BOUND = 16
- NAME_WIDTH = max([len(s) for s,d in linedata])
- if NAME_WIDTH < NAME_WIDTH_LOWER_BOUND:
- NAME_WIDTH = NAME_WIDTH_LOWER_BOUND
- else:
- NAME_WIDTH = NAME_WIDTH_UPPER_BOUND
-
- DOC_WIDTH = WIDTH - NAME_WIDTH - SPACING
- for namestr, doc in linedata:
- line = indent + namestr
- if len(namestr) <= NAME_WIDTH:
- line += ' ' * (NAME_WIDTH + SPACING - len(namestr))
- else:
- lines.append(line)
- line = indent + ' ' * (NAME_WIDTH + SPACING)
- line += _summarize_doc(doc, DOC_WIDTH)
- lines.append(line.rstrip())
- return lines
-
-def _summarize_doc(doc, length=60):
- r"""Parse out a short one line summary from the given doclines.
-
- "doc" is the doc string to summarize.
- "length" is the max length for the summary
-
- >>> _summarize_doc("this function does this")
- 'this function does this'
- >>> _summarize_doc("this function does this", 10)
- 'this fu...'
- >>> _summarize_doc("this function does this\nand that")
- 'this function does this and that'
- >>> _summarize_doc("this function does this\n\nand that")
- 'this function does this'
- """
- import re
- if doc is None:
- return ""
- assert length > 3, "length <= 3 is absurdly short for a doc summary"
- doclines = doc.strip().splitlines(0)
- if not doclines:
- return ""
-
- summlines = []
- for i, line in enumerate(doclines):
- stripped = line.strip()
- if not stripped:
- break
- summlines.append(stripped)
- if len(''.join(summlines)) >= length:
- break
-
- summary = ' '.join(summlines)
- if len(summary) > length:
- summary = summary[:length-3] + "..."
- return summary
-
-
-def line2argv(line):
- r"""Parse the given line into an argument vector.
-
- "line" is the line of input to parse.
-
- This may get niggly when dealing with quoting and escaping. The
- current state of this parsing may not be completely thorough/correct
- in this respect.
-
- >>> from cmdln import line2argv
- >>> line2argv("foo")
- ['foo']
- >>> line2argv("foo bar")
- ['foo', 'bar']
- >>> line2argv("foo bar ")
- ['foo', 'bar']
- >>> line2argv(" foo bar")
- ['foo', 'bar']
-
- Quote handling:
-
- >>> line2argv("'foo bar'")
- ['foo bar']
- >>> line2argv('"foo bar"')
- ['foo bar']
- >>> line2argv(r'"foo\"bar"')
- ['foo"bar']
- >>> line2argv("'foo bar' spam")
- ['foo bar', 'spam']
- >>> line2argv("'foo 'bar spam")
- ['foo bar', 'spam']
-
- >>> line2argv('some\tsimple\ttests')
- ['some', 'simple', 'tests']
- >>> line2argv('a "more complex" test')
- ['a', 'more complex', 'test']
- >>> line2argv('a more="complex test of " quotes')
- ['a', 'more=complex test of ', 'quotes']
- >>> line2argv('a more" complex test of " quotes')
- ['a', 'more complex test of ', 'quotes']
- >>> line2argv('an "embedded \\"quote\\""')
- ['an', 'embedded "quote"']
-
- # Komodo bug 48027
- >>> line2argv('foo bar C:\\')
- ['foo', 'bar', 'C:\\']
-
- # Komodo change 127581
- >>> line2argv(r'"\test\slash" "foo bar" "foo\"bar"')
- ['\\test\\slash', 'foo bar', 'foo"bar']
-
- # Komodo change 127629
- >>> if sys.platform == "win32":
- ... line2argv(r'\foo\bar') == ['\\foo\\bar']
- ... line2argv(r'\\foo\\bar') == ['\\\\foo\\\\bar']
- ... line2argv('"foo') == ['foo']
- ... else:
- ... line2argv(r'\foo\bar') == ['foobar']
- ... line2argv(r'\\foo\\bar') == ['\\foo\\bar']
- ... try:
- ... line2argv('"foo')
- ... except ValueError, ex:
- ... "not terminated" in str(ex)
- True
- True
- True
- """
- import string
- line = line.strip()
- argv = []
- state = "default"
- arg = None # the current argument being parsed
- i = -1
- while 1:
- i += 1
- if i >= len(line): break
- ch = line[i]
-
- if ch == "\\" and i+1 < len(line):
- # escaped char always added to arg, regardless of state
- if arg is None: arg = ""
- if (sys.platform == "win32"
- or state in ("double-quoted", "single-quoted")
- ) and line[i+1] not in tuple('"\''):
- arg += ch
- i += 1
- arg += line[i]
- continue
-
- if state == "single-quoted":
- if ch == "'":
- state = "default"
- else:
- arg += ch
- elif state == "double-quoted":
- if ch == '"':
- state = "default"
- else:
- arg += ch
- elif state == "default":
- if ch == '"':
- if arg is None: arg = ""
- state = "double-quoted"
- elif ch == "'":
- if arg is None: arg = ""
- state = "single-quoted"
- elif ch in string.whitespace:
- if arg is not None:
- argv.append(arg)
- arg = None
- else:
- if arg is None: arg = ""
- arg += ch
- if arg is not None:
- argv.append(arg)
- if not sys.platform == "win32" and state != "default":
- raise ValueError("command line is not terminated: unfinished %s "
- "segment" % state)
- return argv
-
-
-def argv2line(argv):
- r"""Put together the given argument vector into a command line.
-
- "argv" is the argument vector to process.
-
- >>> from cmdln import argv2line
- >>> argv2line(['foo'])
- 'foo'
- >>> argv2line(['foo', 'bar'])
- 'foo bar'
- >>> argv2line(['foo', 'bar baz'])
- 'foo "bar baz"'
- >>> argv2line(['foo"bar'])
- 'foo"bar'
- >>> print argv2line(['foo" bar'])
- 'foo" bar'
- >>> print argv2line(["foo' bar"])
- "foo' bar"
- >>> argv2line(["foo'bar"])
- "foo'bar"
- """
- escapedArgs = []
- for arg in argv:
- if ' ' in arg and '"' not in arg:
- arg = '"'+arg+'"'
- elif ' ' in arg and "'" not in arg:
- arg = "'"+arg+"'"
- elif ' ' in arg:
- arg = arg.replace('"', r'\"')
- arg = '"'+arg+'"'
- escapedArgs.append(arg)
- return ' '.join(escapedArgs)
-
-
-# Recipe: dedent (0.1) in /Users/trentm/tm/recipes/cookbook
-def _dedentlines(lines, tabsize=8, skip_first_line=False):
- """_dedentlines(lines, tabsize=8, skip_first_line=False) -> dedented lines
-
- "lines" is a list of lines to dedent.
- "tabsize" is the tab width to use for indent width calculations.
- "skip_first_line" is a boolean indicating if the first line should
- be skipped for calculating the indent width and for dedenting.
- This is sometimes useful for docstrings and similar.
-
- Same as dedent() except operates on a sequence of lines. Note: the
- lines list is modified **in-place**.
- """
- DEBUG = False
- if DEBUG:
- print "dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\
- % (tabsize, skip_first_line)
- indents = []
- margin = None
- for i, line in enumerate(lines):
- if i == 0 and skip_first_line: continue
- indent = 0
- for ch in line:
- if ch == ' ':
- indent += 1
- elif ch == '\t':
- indent += tabsize - (indent % tabsize)
- elif ch in '\r\n':
- continue # skip all-whitespace lines
- else:
- break
- else:
- continue # skip all-whitespace lines
- if DEBUG: print "dedent: indent=%d: %r" % (indent, line)
- if margin is None:
- margin = indent
- else:
- margin = min(margin, indent)
- if DEBUG: print "dedent: margin=%r" % margin
-
- if margin is not None and margin > 0:
- for i, line in enumerate(lines):
- if i == 0 and skip_first_line: continue
- removed = 0
- for j, ch in enumerate(line):
- if ch == ' ':
- removed += 1
- elif ch == '\t':
- removed += tabsize - (removed % tabsize)
- elif ch in '\r\n':
- if DEBUG: print "dedent: %r: EOL -> strip up to EOL" % line
- lines[i] = lines[i][j:]
- break
- else:
- raise ValueError("unexpected non-whitespace char %r in "
- "line %r while removing %d-space margin"
- % (ch, line, margin))
- if DEBUG:
- print "dedent: %r: %r -> removed %d/%d"\
- % (line, ch, removed, margin)
- if removed == margin:
- lines[i] = lines[i][j+1:]
- break
- elif removed > margin:
- lines[i] = ' '*(removed-margin) + lines[i][j+1:]
- break
- return lines
-
-def _dedent(text, tabsize=8, skip_first_line=False):
- """_dedent(text, tabsize=8, skip_first_line=False) -> dedented text
-
- "text" is the text to dedent.
- "tabsize" is the tab width to use for indent width calculations.
- "skip_first_line" is a boolean indicating if the first line should
- be skipped for calculating the indent width and for dedenting.
- This is sometimes useful for docstrings and similar.
-
- textwrap.dedent(s), but don't expand tabs to spaces
- """
- lines = text.splitlines(1)
- _dedentlines(lines, tabsize=tabsize, skip_first_line=skip_first_line)
- return ''.join(lines)
-
-
-def _get_indent(marker, s, tab_width=8):
- """_get_indent(marker, s, tab_width=8) ->
- (<indentation-of-'marker'>, <indentation-width>)"""
- # Figure out how much the marker is indented.
- INDENT_CHARS = tuple(' \t')
- start = s.index(marker)
- i = start
- while i > 0:
- if s[i-1] not in INDENT_CHARS:
- break
- i -= 1
- indent = s[i:start]
- indent_width = 0
- for ch in indent:
- if ch == ' ':
- indent_width += 1
- elif ch == '\t':
- indent_width += tab_width - (indent_width % tab_width)
- return indent, indent_width
-
-def _get_trailing_whitespace(marker, s):
- """Return the whitespace content trailing the given 'marker' in string 's',
- up to and including a newline.
- """
- suffix = ''
- start = s.index(marker) + len(marker)
- i = start
- while i < len(s):
- if s[i] in ' \t':
- suffix += s[i]
- elif s[i] in '\r\n':
- suffix += s[i]
- if s[i] == '\r' and i+1 < len(s) and s[i+1] == '\n':
- suffix += s[i+1]
- break
- else:
- break
- i += 1
- return suffix
-
-
-
-#---- bash completion support
-# Note: This is still experimental. I expect to change this
-# significantly.
-#
-# To get Bash completion for a cmdln.Cmdln class, run the following
-# bash command:
-# $ complete -C 'python -m cmdln /path/to/script.py CmdlnClass' cmdname
-# For example:
-# $ complete -C 'python -m cmdln ~/bin/svn.py SVN' svn
-#
-#TODO: Simplify the above so don't have to given path to script (try to
-# find it on PATH, if possible). Could also make class name
-# optional if there is only one in the module (common case).
-
-if __name__ == "__main__" and len(sys.argv) == 6:
- def _log(s):
- return # no-op, comment out for debugging
- from os.path import expanduser
- fout = open(expanduser("~/tmp/bashcpln.log"), 'a')
- fout.write(str(s) + '\n')
- fout.close()
-
- # Recipe: module_from_path (1.0.1+)
- def _module_from_path(path):
- import imp, os, sys
- path = os.path.expanduser(path)
- dir = os.path.dirname(path) or os.curdir
- name = os.path.splitext(os.path.basename(path))[0]
- sys.path.insert(0, dir)
- try:
- iinfo = imp.find_module(name, [dir])
- return imp.load_module(name, *iinfo)
- finally:
- sys.path.remove(dir)
-
- def _get_bash_cplns(script_path, class_name, cmd_name,
- token, preceding_token):
- _log('--')
- _log('get_cplns(%r, %r, %r, %r, %r)'
- % (script_path, class_name, cmd_name, token, preceding_token))
- comp_line = os.environ["COMP_LINE"]
- comp_point = int(os.environ["COMP_POINT"])
- _log("COMP_LINE: %r" % comp_line)
- _log("COMP_POINT: %r" % comp_point)
-
- try:
- script = _module_from_path(script_path)
- except ImportError, ex:
- _log("error importing `%s': %s" % (script_path, ex))
- return []
- shell = getattr(script, class_name)()
- cmd_map = shell._get_canonical_map()
- del cmd_map["EOF"]
-
- # Determine if completing the sub-command name.
- parts = comp_line[:comp_point].split(None, 1)
- _log(parts)
- if len(parts) == 1 or not (' ' in parts[1] or '\t' in parts[1]):
- #TODO: if parts[1].startswith('-'): handle top-level opts
- _log("complete sub-command names")
- matches = {}
- for name, canon_name in cmd_map.items():
- if name.startswith(token):
- matches[name] = canon_name
- if not matches:
- return []
- elif len(matches) == 1:
- return matches.keys()
- elif len(set(matches.values())) == 1:
- return [matches.values()[0]]
- else:
- return matches.keys()
-
- # Otherwise, complete options for the given sub-command.
- #TODO: refine this so it does the right thing with option args
- if token.startswith('-'):
- cmd_name = comp_line.split(None, 2)[1]
- try:
- cmd_canon_name = cmd_map[cmd_name]
- except KeyError:
- return []
- handler = shell._get_cmd_handler(cmd_canon_name)
- optparser = getattr(handler, "optparser", None)
- if optparser is None:
- optparser = SubCmdOptionParser()
- opt_strs = []
- for option in optparser.option_list:
- for opt_str in option._short_opts + option._long_opts:
- if opt_str.startswith(token):
- opt_strs.append(opt_str)
- return opt_strs
-
- return []
-
- for cpln in _get_bash_cplns(*sys.argv[1:]):
- print cpln
-
diff --git a/scripts/lib/wic/utils/errors.py b/scripts/lib/wic/utils/errors.py
deleted file mode 100644
index 9410311875..0000000000
--- a/scripts/lib/wic/utils/errors.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env python -tt
-#
-# Copyright (c) 2007 Red Hat, Inc.
-# Copyright (c) 2011 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-class CreatorError(Exception):
- """An exception base class for all imgcreate errors."""
- keyword = '<creator>'
-
- def __init__(self, msg):
- self.msg = msg
-
- def __str__(self):
- if isinstance(self.msg, unicode):
- self.msg = self.msg.encode('utf-8', 'ignore')
- else:
- self.msg = str(self.msg)
- return self.keyword + self.msg
-
-class Usage(CreatorError):
- keyword = '<usage>'
-
- def __str__(self):
- if isinstance(self.msg, unicode):
- self.msg = self.msg.encode('utf-8', 'ignore')
- else:
- self.msg = str(self.msg)
- return self.keyword + self.msg + ', please use "--help" for more info'
-
-class KsError(CreatorError):
- keyword = '<kickstart>'
-
-class ImageError(CreatorError):
- keyword = '<mount>'
diff --git a/scripts/lib/wic/utils/fs_related.py b/scripts/lib/wic/utils/fs_related.py
deleted file mode 100644
index ea9f85c60f..0000000000
--- a/scripts/lib/wic/utils/fs_related.py
+++ /dev/null
@@ -1,111 +0,0 @@
-#!/usr/bin/env python -tt
-#
-# Copyright (c) 2007, Red Hat, Inc.
-# Copyright (c) 2009, 2010, 2011 Intel, Inc.
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-from __future__ import with_statement
-import os
-import sys
-import errno
-import stat
-import random
-import string
-import time
-import uuid
-
-from wic import msger
-from wic.utils import runner
-from wic.utils.errors import *
-from wic.utils.oe.misc import *
-
-def find_binary_path(binary):
- if os.environ.has_key("PATH"):
- paths = os.environ["PATH"].split(":")
- else:
- paths = []
- if os.environ.has_key("HOME"):
- paths += [os.environ["HOME"] + "/bin"]
- paths += ["/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"]
-
- for path in paths:
- bin_path = "%s/%s" % (path, binary)
- if os.path.exists(bin_path):
- return bin_path
-
- print "External command '%s' not found, exiting." % binary
- print " (Please install '%s' on your host system)" % binary
- sys.exit(1)
-
-def makedirs(dirname):
- """A version of os.makedirs() that doesn't throw an
- exception if the leaf directory already exists.
- """
- try:
- os.makedirs(dirname)
- except OSError, err:
- if err.errno != errno.EEXIST:
- raise
-
-class Disk:
- """
- Generic base object for a disk.
- """
- def __init__(self, size, device = None):
- self._device = device
- self._size = size
-
- def create(self):
- pass
-
- def cleanup(self):
- pass
-
- def get_device(self):
- return self._device
- def set_device(self, path):
- self._device = path
- device = property(get_device, set_device)
-
- def get_size(self):
- return self._size
- size = property(get_size)
-
-
-class DiskImage(Disk):
- """
- A Disk backed by a file.
- """
- def __init__(self, image_file, size):
- Disk.__init__(self, size)
- self.image_file = image_file
-
- def exists(self):
- return os.path.exists(self.image_file)
-
- def create(self):
- if self.device is not None:
- return
-
- blocks = self.size / 1024
- if self.size - blocks * 1024:
- blocks += 1
-
- # create disk image
- dd_cmd = "dd if=/dev/zero of=%s bs=1024 seek=%d count=1" % \
- (self.image_file, blocks)
- exec_cmd(dd_cmd)
-
- self.device = self.image_file
diff --git a/scripts/lib/wic/utils/misc.py b/scripts/lib/wic/utils/misc.py
index 6e56316608..46099840ca 100644
--- a/scripts/lib/wic/utils/misc.py
+++ b/scripts/lib/wic/utils/misc.py
@@ -1,59 +1,230 @@
-#!/usr/bin/env python -tt
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
-# Copyright (c) 2010, 2011 Intel Inc.
+# Copyright (c) 2013, Intel Corporation.
+# All rights reserved.
#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# DESCRIPTION
+# This module provides a place to collect various wic-related utils
+# for the OpenEmbedded Image Tools.
+#
+# AUTHORS
+# Tom Zanussi <tom.zanussi (at] linux.intel.com>
+#
+"""Miscellaneous functions."""
+import logging
import os
-import sys
-import time
+import re
-def build_name(kscfg, release=None, prefix = None, suffix = None):
- """Construct and return an image name string.
+from collections import defaultdict
+from distutils import spawn
- This is a utility function to help create sensible name and fslabel
- strings. The name is constructed using the sans-prefix-and-extension
- kickstart filename and the supplied prefix and suffix.
+from wic import WicError
+from wic.utils import runner
- kscfg -- a path to a kickstart file
- release -- a replacement to suffix for image release
- prefix -- a prefix to prepend to the name; defaults to None, which causes
- no prefix to be used
- suffix -- a suffix to append to the name; defaults to None, which causes
- a YYYYMMDDHHMM suffix to be used
+logger = logging.getLogger('wic')
- Note, if maxlen is less then the len(suffix), you get to keep both pieces.
+# executable -> recipe pairs for exec_native_cmd
+NATIVE_RECIPES = {"bmaptool": "bmap-tools",
+ "grub-mkimage": "grub-efi",
+ "isohybrid": "syslinux",
+ "mcopy": "mtools",
+ "mkdosfs": "dosfstools",
+ "mkisofs": "cdrtools",
+ "mkfs.btrfs": "btrfs-tools",
+ "mkfs.ext2": "e2fsprogs",
+ "mkfs.ext3": "e2fsprogs",
+ "mkfs.ext4": "e2fsprogs",
+ "mkfs.vfat": "dosfstools",
+ "mksquashfs": "squashfs-tools",
+ "mkswap": "util-linux",
+ "mmd": "syslinux",
+ "parted": "parted",
+ "sfdisk": "util-linux",
+ "sgdisk": "gptfdisk",
+ "syslinux": "syslinux"
+ }
+
+def _exec_cmd(cmd_and_args, as_shell=False):
+ """
+ Execute command, catching stderr, stdout
+ Need to execute as_shell if the command uses wildcards
"""
- name = os.path.basename(kscfg)
- idx = name.rfind('.')
- if idx >= 0:
- name = name[:idx]
+ logger.debug("_exec_cmd: %s", cmd_and_args)
+ args = cmd_and_args.split()
+ logger.debug(args)
+
+ if as_shell:
+ ret, out = runner.runtool(cmd_and_args)
+ else:
+ ret, out = runner.runtool(args)
+ out = out.strip()
+ if ret != 0:
+ raise WicError("_exec_cmd: %s returned '%s' instead of 0\noutput: %s" % \
+ (cmd_and_args, ret, out))
+
+ logger.debug("_exec_cmd: output for %s (rc = %d): %s",
+ cmd_and_args, ret, out)
+
+ return ret, out
+
+
+def exec_cmd(cmd_and_args, as_shell=False):
+ """
+ Execute command, return output
+ """
+ return _exec_cmd(cmd_and_args, as_shell)[1]
+
+
+def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""):
+ """
+ Execute native command, catching stderr, stdout
+
+ Need to execute as_shell if the command uses wildcards
+
+ Always need to execute native commands as_shell
+ """
+ # The reason -1 is used is because there may be "export" commands.
+ args = cmd_and_args.split(';')[-1].split()
+ logger.debug(args)
+
+ if pseudo:
+ cmd_and_args = pseudo + cmd_and_args
- if release is not None:
- suffix = ""
- if prefix is None:
- prefix = ""
- if suffix is None:
- suffix = time.strftime("%Y%m%d%H%M")
+ wtools_sysroot = get_bitbake_var("RECIPE_SYSROOT_NATIVE", "wic-tools")
- if name.startswith(prefix):
- name = name[len(prefix):]
+ native_paths = \
+ "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/sbin:%s/usr/sbin:%s/usr/bin" % \
+ (wtools_sysroot, wtools_sysroot, wtools_sysroot,
+ native_sysroot, native_sysroot, native_sysroot)
+ native_cmd_and_args = "export PATH=%s:$PATH;%s" % \
+ (native_paths, cmd_and_args)
+ logger.debug("exec_native_cmd: %s", native_cmd_and_args)
- prefix = "%s-" % prefix if prefix else ""
- suffix = "-%s" % suffix if suffix else ""
+ # If the command isn't in the native sysroot say we failed.
+ if spawn.find_executable(args[0], native_paths):
+ ret, out = _exec_cmd(native_cmd_and_args, True)
+ else:
+ ret = 127
+ out = "can't find native executable %s in %s" % (args[0], native_paths)
- ret = prefix + name + suffix
+ prog = args[0]
+ # shell command-not-found
+ if ret == 127 \
+ or (pseudo and ret == 1 and out == "Can't find '%s' in $PATH." % prog):
+ msg = "A native program %s required to build the image "\
+ "was not found (see details above).\n\n" % prog
+ recipe = NATIVE_RECIPES.get(prog)
+ if recipe:
+ msg += "Please make sure wic-tools have %s-native in its DEPENDS, bake it with 'bitbake wic-tools' "\
+ "and try again.\n" % recipe
+ else:
+ msg += "Wic failed to find a recipe to build native %s. Please "\
+ "file a bug against wic.\n" % prog
+ raise WicError(msg)
- return ret
+ return ret, out
+
+BOOTDD_EXTRA_SPACE = 16384
+
+class BitbakeVars(defaultdict):
+ """
+ Container for Bitbake variables.
+ """
+ def __init__(self):
+ defaultdict.__init__(self, dict)
+
+ # default_image and vars_dir attributes should be set from outside
+ self.default_image = None
+ self.vars_dir = None
+
+ def _parse_line(self, line, image, matcher=re.compile(r"^(\w+)=(.+)")):
+ """
+ Parse one line from bitbake -e output or from .env file.
+ Put result key-value pair into the storage.
+ """
+ if "=" not in line:
+ return
+ match = matcher.match(line)
+ if not match:
+ return
+ key, val = match.groups()
+ self[image][key] = val.strip('"')
+
+ def get_var(self, var, image=None, cache=True):
+ """
+ Get bitbake variable from 'bitbake -e' output or from .env file.
+ This is a lazy method, i.e. it runs bitbake or parses file only when
+ only when variable is requested. It also caches results.
+ """
+ if not image:
+ image = self.default_image
+
+ if image not in self:
+ if image and self.vars_dir:
+ fname = os.path.join(self.vars_dir, image + '.env')
+ if os.path.isfile(fname):
+ # parse .env file
+ with open(fname) as varsfile:
+ for line in varsfile:
+ self._parse_line(line, image)
+ else:
+ print("Couldn't get bitbake variable from %s." % fname)
+ print("File %s doesn't exist." % fname)
+ return
+ else:
+ # Get bitbake -e output
+ cmd = "bitbake -e"
+ if image:
+ cmd += " %s" % image
+
+ log_level = logger.getEffectiveLevel()
+ logger.setLevel(logging.INFO)
+ ret, lines = _exec_cmd(cmd)
+ logger.setLevel(log_level)
+
+ if ret:
+ logger.error("Couldn't get '%s' output.", cmd)
+ logger.error("Bitbake failed with error:\n%s\n", lines)
+ return
+
+ # Parse bitbake -e output
+ for line in lines.split('\n'):
+ self._parse_line(line, image)
+
+ # Make first image a default set of variables
+ if cache:
+ images = [key for key in self if key]
+ if len(images) == 1:
+ self[None] = self[image]
+
+ result = self[image].get(var)
+ if not cache:
+ self.pop(image, None)
+
+ return result
+
+# Create BB_VARS singleton
+BB_VARS = BitbakeVars()
+
+def get_bitbake_var(var, image=None, cache=True):
+ """
+ Provide old get_bitbake_var API by wrapping
+ get_var method of BB_VARS singleton.
+ """
+ return BB_VARS.get_var(var, image, cache)
diff --git a/scripts/lib/wic/utils/oe/__init__.py b/scripts/lib/wic/utils/oe/__init__.py
deleted file mode 100644
index 0a81575a74..0000000000
--- a/scripts/lib/wic/utils/oe/__init__.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# OpenEmbedded wic utils library
-#
-# Copyright (c) 2013, Intel Corporation.
-# All rights reserved.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# AUTHORS
-# Tom Zanussi <tom.zanussi (at] linux.intel.com>
-#
diff --git a/scripts/lib/wic/utils/oe/misc.py b/scripts/lib/wic/utils/oe/misc.py
deleted file mode 100644
index ea9b6e8ec4..0000000000
--- a/scripts/lib/wic/utils/oe/misc.py
+++ /dev/null
@@ -1,203 +0,0 @@
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-#
-# Copyright (c) 2013, Intel Corporation.
-# All rights reserved.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# DESCRIPTION
-# This module provides a place to collect various wic-related utils
-# for the OpenEmbedded Image Tools.
-#
-# AUTHORS
-# Tom Zanussi <tom.zanussi (at] linux.intel.com>
-#
-
-from wic import msger
-from wic.utils import runner
-
-def __exec_cmd(cmd_and_args, as_shell = False, catch = 3):
- """
- Execute command, catching stderr, stdout
-
- Need to execute as_shell if the command uses wildcards
- """
- msger.debug("__exec_cmd: %s" % cmd_and_args)
- args = cmd_and_args.split()
- msger.debug(args)
-
- if (as_shell):
- rc, out = runner.runtool(cmd_and_args, catch)
- else:
- rc, out = runner.runtool(args, catch)
- out = out.strip()
- msger.debug("__exec_cmd: output for %s (rc = %d): %s" % \
- (cmd_and_args, rc, out))
-
- return (rc, out)
-
-
-def exec_cmd(cmd_and_args, as_shell = False, catch = 3):
- """
- Execute command, catching stderr, stdout
-
- Exits if rc non-zero
- """
- rc, out = __exec_cmd(cmd_and_args, as_shell, catch)
-
- if rc != 0:
- msger.error("exec_cmd: %s returned '%s' instead of 0" % (cmd_and_args, rc))
-
- return out
-
-
-def exec_cmd_quiet(cmd_and_args, as_shell = False):
- """
- Execute command, catching nothing in the output
-
- Exits if rc non-zero
- """
- return exec_cmd(cmd_and_args, as_shell, 0)
-
-
-def exec_native_cmd(cmd_and_args, native_sysroot, catch = 3):
- """
- Execute native command, catching stderr, stdout
-
- Need to execute as_shell if the command uses wildcards
-
- Always need to execute native commands as_shell
- """
- native_paths = \
- "export PATH=%s/sbin:%s/usr/sbin:%s/usr/bin:$PATH" % \
- (native_sysroot, native_sysroot, native_sysroot)
- native_cmd_and_args = "%s;%s" % (native_paths, cmd_and_args)
- msger.debug("exec_native_cmd: %s" % cmd_and_args)
-
- args = cmd_and_args.split()
- msger.debug(args)
-
- rc, out = __exec_cmd(native_cmd_and_args, True, catch)
-
- if rc == 127: # shell command-not-found
- msger.error("A native (host) program required to build the image "
- "was not found (see details above). Please make sure "
- "it's installed and try again.")
-
- return (rc, out)
-
-
-def exec_native_cmd_quiet(cmd_and_args, native_sysroot):
- """
- Execute native command, catching nothing in the output
-
- Need to execute as_shell if the command uses wildcards
-
- Always need to execute native commands as_shell
- """
- return exec_native_cmd(cmd_and_args, native_sysroot, 0)
-
-
-# kickstart doesn't support variable substution in commands, so this
-# is our current simplistic scheme for supporting that
-
-wks_vars = dict()
-
-def get_wks_var(key):
- return wks_vars[key]
-
-def add_wks_var(key, val):
- wks_vars[key] = val
-
-BOOTDD_EXTRA_SPACE = 16384
-
-__bitbake_env_lines = ""
-
-def set_bitbake_env_lines(bitbake_env_lines):
- global __bitbake_env_lines
- __bitbake_env_lines = bitbake_env_lines
-
-def get_bitbake_env_lines():
- return __bitbake_env_lines
-
-def find_bitbake_env_lines(image_name):
- """
- If image_name is empty, plugins might still be able to use the
- environment, so set it regardless.
- """
- if image_name:
- bitbake_env_cmd = "bitbake -e %s" % image_name
- else:
- bitbake_env_cmd = "bitbake -e"
- rc, bitbake_env_lines = __exec_cmd(bitbake_env_cmd)
- if rc != 0:
- print "Couldn't get '%s' output." % bitbake_env_cmd
- return None
-
- return bitbake_env_lines
-
-def find_artifact(bitbake_env_lines, variable):
- """
- Gather the build artifact for the current image (the image_name
- e.g. core-image-minimal) for the current MACHINE set in local.conf
- """
- retval = ""
-
- for line in bitbake_env_lines.split('\n'):
- if (get_line_val(line, variable)):
- retval = get_line_val(line, variable)
- break
-
- return retval
-
-def get_line_val(line, key):
- """
- Extract the value from the VAR="val" string
- """
- if line.startswith(key + "="):
- stripped_line = line.split('=')[1]
- stripped_line = stripped_line.replace('\"', '')
- return stripped_line
- return None
-
-def get_bitbake_var(key):
- for line in __bitbake_env_lines.split('\n'):
- if (get_line_val(line, key)):
- val = get_line_val(line, key)
- return val
- return None
-
-def parse_sourceparams(sourceparams):
- """
- Split sourceparams string of the form key1=val1[,key2=val2,...]
- into a dict. Also accepts valueless keys i.e. without =.
-
- Returns dict of param key/val pairs (note that val may be None).
- """
- params_dict = {}
-
- params = sourceparams.split(',')
- if params:
- for p in params:
- if not p:
- continue
- if not '=' in p:
- key = p
- val = None
- else:
- key, val = p.split('=')
- params_dict[key] = val
-
- return params_dict
diff --git a/scripts/lib/wic/utils/partitionedfs.py b/scripts/lib/wic/utils/partitionedfs.py
deleted file mode 100644
index 162f8e1b9c..0000000000
--- a/scripts/lib/wic/utils/partitionedfs.py
+++ /dev/null
@@ -1,375 +0,0 @@
-#!/usr/bin/env python -tt
-#
-# Copyright (c) 2009, 2010, 2011 Intel, Inc.
-# Copyright (c) 2007, 2008 Red Hat, Inc.
-# Copyright (c) 2008 Daniel P. Berrange
-# Copyright (c) 2008 David P. Huff
-#
-# This program is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License as published by the Free
-# Software Foundation; version 2 of the License
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
-# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc., 59
-# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-import os
-
-from wic import msger
-from wic.utils import runner
-from wic.utils.errors import ImageError
-from wic.utils.fs_related import *
-from wic.utils.oe.misc import *
-
-# Overhead of the MBR partitioning scheme (just one sector)
-MBR_OVERHEAD = 1
-
-# Overhead of the GPT partitioning scheme
-GPT_OVERHEAD = 34
-
-# Size of a sector in bytes
-SECTOR_SIZE = 512
-
-class Image:
- """
- Generic base object for an image.
-
- An Image is a container for a set of DiskImages and associated
- partitions.
- """
- def __init__(self):
- self.disks = {}
- self.partitions = []
- self.parted = find_binary_path("parted")
- # Size of a sector used in calculations
- self.sector_size = SECTOR_SIZE
- self._partitions_layed_out = False
-
- def __add_disk(self, disk_name):
- """ Add a disk 'disk_name' to the internal list of disks. Note,
- 'disk_name' is the name of the disk in the target system
- (e.g., sdb). """
-
- if disk_name in self.disks:
- # We already have this disk
- return
-
- assert not self._partitions_layed_out
-
- self.disks[disk_name] = \
- { 'disk': None, # Disk object
- 'numpart': 0, # Number of allocate partitions
- 'realpart': 0, # Number of partitions in the partition table
- 'partitions': [], # Indexes to self.partitions
- 'offset': 0, # Offset of next partition (in sectors)
- # Minimum required disk size to fit all partitions (in bytes)
- 'min_size': 0,
- 'ptable_format': "msdos" } # Partition table format
-
- def add_disk(self, disk_name, disk_obj):
- """ Add a disk object which have to be partitioned. More than one disk
- can be added. In case of multiple disks, disk partitions have to be
- added for each disk separately with 'add_partition()". """
-
- self.__add_disk(disk_name)
- self.disks[disk_name]['disk'] = disk_obj
-
- def __add_partition(self, part):
- """ This is a helper function for 'add_partition()' which adds a
- partition to the internal list of partitions. """
-
- assert not self._partitions_layed_out
-
- self.partitions.append(part)
- self.__add_disk(part['disk_name'])
-
- def add_partition(self, size, disk_name, mountpoint, source_file = None, fstype = None,
- label=None, fsopts = None, boot = False, align = None, no_table=False,
- part_type = None):
- """ Add the next partition. Prtitions have to be added in the
- first-to-last order. """
-
- ks_pnum = len(self.partitions)
-
- # Converting kB to sectors for parted
- size = size * 1024 / self.sector_size
-
- # We still need partition for "/" or non-subvolume
- if mountpoint == "/" or not fsopts:
- part = { 'ks_pnum' : ks_pnum, # Partition number in the KS file
- 'size': size, # In sectors
- 'mountpoint': mountpoint, # Mount relative to chroot
- 'source_file': source_file, # partition contents
- 'fstype': fstype, # Filesystem type
- 'fsopts': fsopts, # Filesystem mount options
- 'label': label, # Partition label
- 'disk_name': disk_name, # physical disk name holding partition
- 'device': None, # kpartx device node for partition
- 'num': None, # Partition number
- 'boot': boot, # Bootable flag
- 'align': align, # Partition alignment
- 'no_table' : no_table, # Partition does not appear in partition table
- 'part_type' : part_type } # Partition type
-
- self.__add_partition(part)
-
- def layout_partitions(self, ptable_format = "msdos"):
- """ Layout the partitions, meaning calculate the position of every
- partition on the disk. The 'ptable_format' parameter defines the
- partition table format and may be "msdos". """
-
- msger.debug("Assigning %s partitions to disks" % ptable_format)
-
- if ptable_format not in ('msdos', 'gpt'):
- raise ImageError("Unknown partition table format '%s', supported " \
- "formats are: 'msdos'" % ptable_format)
-
- if self._partitions_layed_out:
- return
-
- self._partitions_layed_out = True
-
- # Go through partitions in the order they are added in .ks file
- for n in range(len(self.partitions)):
- p = self.partitions[n]
-
- if not self.disks.has_key(p['disk_name']):
- raise ImageError("No disk %s for partition %s" \
- % (p['disk_name'], p['mountpoint']))
-
- if p['part_type']:
- # The --part-type can also be implemented for MBR partitions,
- # in which case it would map to the 1-byte "partition type"
- # filed at offset 3 of the partition entry.
- raise ImageError("setting custom partition type is not " \
- "implemented for msdos partitions")
-
- # Get the disk where the partition is located
- d = self.disks[p['disk_name']]
- d['numpart'] += 1
- if not p['no_table']:
- d['realpart'] += 1
- d['ptable_format'] = ptable_format
-
- if d['numpart'] == 1:
- if ptable_format == "msdos":
- overhead = MBR_OVERHEAD
- elif ptable_format == "gpt":
- overhead = GPT_OVERHEAD
-
- # Skip one sector required for the partitioning scheme overhead
- d['offset'] += overhead
-
- if d['realpart'] > 3:
- # Reserve a sector for EBR for every logical partition
- # before alignment is performed.
- if ptable_format == "msdos":
- d['offset'] += 1
-
-
- if p['align']:
- # If not first partition and we do have alignment set we need
- # to align the partition.
- # FIXME: This leaves a empty spaces to the disk. To fill the
- # gaps we could enlargea the previous partition?
-
- # Calc how much the alignment is off.
- align_sectors = d['offset'] % (p['align'] * 1024 / self.sector_size)
-
- if align_sectors:
- # If partition is not aligned as required, we need
- # to move forward to the next alignment point
- align_sectors = (p['align'] * 1024 / self.sector_size) - align_sectors
-
- msger.debug("Realignment for %s%s with %s sectors, original"
- " offset %s, target alignment is %sK." %
- (p['disk_name'], d['numpart'], align_sectors,
- d['offset'], p['align']))
-
- # increase the offset so we actually start the partition on right alignment
- d['offset'] += align_sectors
-
- p['start'] = d['offset']
- d['offset'] += p['size']
-
- p['type'] = 'primary'
- if not p['no_table']:
- p['num'] = d['realpart']
- else:
- p['num'] = 0
-
- if d['ptable_format'] == "msdos":
- if d['realpart'] > 3:
- p['type'] = 'logical'
- p['num'] = d['realpart'] + 1
-
- d['partitions'].append(n)
- msger.debug("Assigned %s to %s%d, sectors range %d-%d size %d "
- "sectors (%d bytes)." \
- % (p['mountpoint'], p['disk_name'], p['num'],
- p['start'], p['start'] + p['size'] - 1,
- p['size'], p['size'] * self.sector_size))
-
- # Once all the partitions have been layed out, we can calculate the
- # minumim disk sizes.
- for disk_name, d in self.disks.items():
- d['min_size'] = d['offset']
- if d['ptable_format'] == "gpt":
- d['min_size'] += GPT_OVERHEAD
-
- d['min_size'] *= self.sector_size
-
- def __run_parted(self, args):
- """ Run parted with arguments specified in the 'args' list. """
-
- args.insert(0, self.parted)
- msger.debug(args)
-
- rc, out = runner.runtool(args, catch = 3)
- out = out.strip()
- if out:
- msger.debug('"parted" output: %s' % out)
-
- if rc != 0:
- # We don't throw exception when return code is not 0, because
- # parted always fails to reload part table with loop devices. This
- # prevents us from distinguishing real errors based on return
- # code.
- msger.error("WARNING: parted returned '%s' instead of 0 (use --debug for details)" % rc)
-
- def __create_partition(self, device, parttype, fstype, start, size):
- """ Create a partition on an image described by the 'device' object. """
-
- # Start is included to the size so we need to substract one from the end.
- end = start + size - 1
- msger.debug("Added '%s' partition, sectors %d-%d, size %d sectors" %
- (parttype, start, end, size))
-
- args = ["-s", device, "unit", "s", "mkpart", parttype]
- if fstype:
- args.extend([fstype])
- args.extend(["%d" % start, "%d" % end])
-
- return self.__run_parted(args)
-
- def __format_disks(self):
- self.layout_partitions()
-
- for dev in self.disks.keys():
- d = self.disks[dev]
- msger.debug("Initializing partition table for %s" % \
- (d['disk'].device))
- self.__run_parted(["-s", d['disk'].device, "mklabel",
- d['ptable_format']])
-
- msger.debug("Creating partitions")
-
- for p in self.partitions:
- if p['num'] == 0:
- continue
-
- d = self.disks[p['disk_name']]
- if d['ptable_format'] == "msdos" and p['num'] == 5:
- # Create an extended partition (note: extended
- # partition is described in MBR and contains all
- # logical partitions). The logical partitions save a
- # sector for an EBR just before the start of a
- # partition. The extended partition must start one
- # sector before the start of the first logical
- # partition. This way the first EBR is inside of the
- # extended partition. Since the extended partitions
- # starts a sector before the first logical partition,
- # add a sector at the back, so that there is enough
- # room for all logical partitions.
- self.__create_partition(d['disk'].device, "extended",
- None, p['start'] - 1,
- d['offset'] - p['start'] + 1)
-
- if p['fstype'] == "swap":
- parted_fs_type = "linux-swap"
- elif p['fstype'] == "vfat":
- parted_fs_type = "fat32"
- elif p['fstype'] == "msdos":
- parted_fs_type = "fat16"
- else:
- # Type for ext2/ext3/ext4/btrfs
- parted_fs_type = "ext2"
-
- # Boot ROM of OMAP boards require vfat boot partition to have an
- # even number of sectors.
- if p['mountpoint'] == "/boot" and p['fstype'] in ["vfat", "msdos"] \
- and p['size'] % 2:
- msger.debug("Substracting one sector from '%s' partition to " \
- "get even number of sectors for the partition" % \
- p['mountpoint'])
- p['size'] -= 1
-
- self.__create_partition(d['disk'].device, p['type'],
- parted_fs_type, p['start'], p['size'])
-
- if p['boot']:
- flag_name = "boot"
- msger.debug("Set '%s' flag for partition '%s' on disk '%s'" % \
- (flag_name, p['num'], d['disk'].device))
- self.__run_parted(["-s", d['disk'].device, "set",
- "%d" % p['num'], flag_name, "on"])
-
- # Parted defaults to enabling the lba flag for fat16 partitions,
- # which causes compatibility issues with some firmware (and really
- # isn't necessary).
- if parted_fs_type == "fat16":
- if d['ptable_format'] == 'msdos':
- msger.debug("Disable 'lba' flag for partition '%s' on disk '%s'" % \
- (p['num'], d['disk'].device))
- self.__run_parted(["-s", d['disk'].device, "set",
- "%d" % p['num'], "lba", "off"])
-
- def cleanup(self):
- if self.disks:
- for dev in self.disks.keys():
- d = self.disks[dev]
- try:
- d['disk'].cleanup()
- except:
- pass
-
- def __write_partition(self, num, source_file, start, size):
- """
- Install source_file contents into a partition.
- """
- if not source_file: # nothing to write
- return
-
- # Start is included in the size so need to substract one from the end.
- end = start + size - 1
- msger.debug("Installed %s in partition %d, sectors %d-%d, size %d sectors" % (source_file, num, start, end, size))
-
- dd_cmd = "dd if=%s of=%s bs=%d seek=%d count=%d conv=notrunc" % \
- (source_file, self.image_file, self.sector_size, start, size)
- exec_cmd(dd_cmd)
-
-
- def assemble(self, image_file):
- msger.debug("Installing partitions")
-
- self.image_file = image_file
-
- for p in self.partitions:
- d = self.disks[p['disk_name']]
-
- self.__write_partition(p['num'], p['source_file'],
- p['start'], p['size'])
-
- def create(self):
- for dev in self.disks.keys():
- d = self.disks[dev]
- d['disk'].create()
-
- self.__format_disks()
-
- return
diff --git a/scripts/lib/wic/utils/runner.py b/scripts/lib/wic/utils/runner.py
index 2ae9f417c5..4aa00fbe20 100644
--- a/scripts/lib/wic/utils/runner.py
+++ b/scripts/lib/wic/utils/runner.py
@@ -14,29 +14,17 @@
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc., 59
# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-
-import os
import subprocess
-from wic import msger
+from wic import WicError
-def runtool(cmdln_or_args, catch=1):
+def runtool(cmdln_or_args):
""" wrapper for most of the subprocess calls
input:
cmdln_or_args: can be both args and cmdln str (shell=True)
- catch: 0, quitely run
- 1, only STDOUT
- 2, only STDERR
- 3, both STDOUT and STDERR
return:
- (rc, output)
- if catch==0: the output will always None
+ rc, output
"""
-
- if catch not in (0, 1, 2, 3):
- # invalid catch selection, will cause exception, that's good
- return None
-
if isinstance(cmdln_or_args, list):
cmd = cmdln_or_args[0]
shell = False
@@ -45,65 +33,20 @@ def runtool(cmdln_or_args, catch=1):
cmd = shlex.split(cmdln_or_args)[0]
shell = True
- if catch != 3:
- dev_null = os.open("/dev/null", os.O_WRONLY)
-
- if catch == 0:
- sout = dev_null
- serr = dev_null
- elif catch == 1:
- sout = subprocess.PIPE
- serr = dev_null
- elif catch == 2:
- sout = dev_null
- serr = subprocess.PIPE
- elif catch == 3:
- sout = subprocess.PIPE
- serr = subprocess.STDOUT
+ sout = subprocess.PIPE
+ serr = subprocess.STDOUT
try:
- p = subprocess.Popen(cmdln_or_args, stdout=sout,
- stderr=serr, shell=shell)
- (sout, serr) = p.communicate()
- # combine stdout and stderr, filter None out
- out = ''.join(filter(None, [sout, serr]))
- except OSError, e:
- if e.errno == 2:
+ process = subprocess.Popen(cmdln_or_args, stdout=sout,
+ stderr=serr, shell=shell)
+ sout, serr = process.communicate()
+ # combine stdout and stderr, filter None out and decode
+ out = ''.join([out.decode('utf-8') for out in [sout, serr] if out])
+ except OSError as err:
+ if err.errno == 2:
# [Errno 2] No such file or directory
- msger.error('Cannot run command: %s, lost dependency?' % cmd)
+ raise WicError('Cannot run command: %s, lost dependency?' % cmd)
else:
raise # relay
- finally:
- if catch != 3:
- os.close(dev_null)
-
- return (p.returncode, out)
-
-def show(cmdln_or_args):
- # show all the message using msger.verbose
-
- rc, out = runtool(cmdln_or_args, catch=3)
-
- if isinstance(cmdln_or_args, list):
- cmd = ' '.join(cmdln_or_args)
- else:
- cmd = cmdln_or_args
-
- msg = 'running command: "%s"' % cmd
- if out: out = out.strip()
- if out:
- msg += ', with output::'
- msg += '\n +----------------'
- for line in out.splitlines():
- msg += '\n | %s' % line
- msg += '\n +----------------'
-
- msger.verbose(msg)
- return rc
-
-def outs(cmdln_or_args, catch=1):
- # get the outputs of tools
- return runtool(cmdln_or_args, catch)[1].strip()
-def quiet(cmdln_or_args):
- return runtool(cmdln_or_args, catch=0)[0]
+ return process.returncode, out