diff --git a/examples/meter-detection/README.md b/examples/meter-detection/README.md index 4d4ee9b..dacaf65 100644 --- a/examples/meter-detection/README.md +++ b/examples/meter-detection/README.md @@ -12,38 +12,51 @@ https://universe.roboflow.com/kemals-workspace-kbc8l/electric-meter-detection-o4 ## Prerequisites +- Install or sync the project dependencies: `uv sync` +- The virtual environment is activated. - AWS credentials configured for the profile in `config.yaml` -- Infrastructure already deployed with `uv run qc-cli infra setup` -- A Roboflow API key exported as `ROBOFLOW_API_KEY` -- `curl` and `unzip` available locally - -Install or sync the project dependencies: - -```bash -uv sync -``` - -Set the Roboflow API key for the current shell: - -```bash -export ROBOFLOW_API_KEY=your-roboflow-api-key -``` +- Infrastructure already deployed with `qc-cli infra setup` ## 1. Download The Dataset -Download version 1 of the dataset in YOLO format. The script uses the Roboflow REST API directly and does not require Python: +Register or sign in to Roboflow, then open the dataset page: -```bash -bash examples/meter-detection/download_dataset.sh +```text +https://universe.roboflow.com/kemals-workspace-kbc8l/electric-meter-detection-o4tfi/dataset/1 ``` -Confirm the extracted dataset has a YOLO data file and image splits: +Download the dataset in YOLOv26 format from the Roboflow UI, then extract the downloaded archive into: + +```text +examples/meter-detection/data/electric-meter-detection +``` + +The `data.yaml` file should be directly under that folder: + +```text +examples/meter-detection/data/electric-meter-detection/data.yaml +``` + +Do not move `data.yaml` into the `train/` split folder. + +After extracting, confirm the dataset has a YOLO data file and image splits: ```bash find examples/meter-detection/data/electric-meter-detection -maxdepth 2 -type d | sort find examples/meter-detection/data/electric-meter-detection -name data.yaml -print ``` +Open `examples/meter-detection/data/electric-meter-detection/data.yaml` and make sure the split paths are relative to that folder: + +```yaml +path: . +train: train/images +val: valid/images +test: test/images +``` + +If your downloaded dataset does not include a `test/` folder, remove the `test:` line. + The expected layout is similar to: ```text @@ -54,8 +67,6 @@ examples/meter-detection/data/electric-meter-detection/ test/ ``` -The `test/` split may be absent depending on the exported dataset version. - ## 2. Configure SageMaker Training Update `config.yaml` so the training section points at this example's source directory: @@ -107,7 +118,7 @@ sagemaker: Confirm the CLI can see the configured SageMaker role and S3 bucket: ```bash -uv run qc-cli infra status --config config.yaml +qc-cli infra status --config config.yaml ``` ## 4. Upload The Dataset @@ -115,29 +126,31 @@ uv run qc-cli infra status --config config.yaml Upload the downloaded Roboflow dataset to the `s3.data_prefix` configured in `config.yaml`: ```bash -uv run qc-cli upload examples/meter-detection/data/electric-meter-detection --config config.yaml +qc-cli upload examples/meter-detection/data/electric-meter-detection ``` Directory uploads preserve paths relative to the uploaded directory, so SageMaker receives the dataset root with `data.yaml` plus the split directories. +In SageMaker, this uploaded dataset root is mounted at `/opt/ml/input/data/train`. That `train` path is the SageMaker channel name, not the YOLO `train/` split folder. + ## 5. Start Training Submit the SageMaker training job: ```bash -uv run qc-cli train start --config config.yaml +qc-cli train start ``` The command prints the submitted SageMaker job name. Check progress with: ```bash -uv run qc-cli train status --config config.yaml +qc-cli train status ``` Or pass the job name explicitly: ```bash -uv run qc-cli train status qc-cli-YYYYMMDD-HHMMSS --config config.yaml +qc-cli train status qc-cli-YYYYMMDD-HHMMSS ``` ## Outputs @@ -167,6 +180,7 @@ Values under `sagemaker.training.hyperparameters` are passed to `source/train.py | `workers` | int | `2` | DataLoader worker count. | | `patience` | int | `20` | Early stopping patience. | | `device` | string | auto | Optional Ultralytics device value such as `0` or `cpu`. | -| `data-yaml` | string | auto | Optional path to `data.yaml`; normally discovered from `SM_CHANNEL_TRAIN`. | +| `data-yaml` | string | auto | Optional path to `data.yaml`; normally discovered from the uploaded dataset root. | +| `dataset-dir` | string | `SM_CHANNEL_TRAIN` | Uploaded dataset root mounted by SageMaker. | -Do not set `train-dir` or `model-dir` in normal SageMaker runs. SageMaker sets those automatically through `SM_CHANNEL_TRAIN` and `SM_MODEL_DIR`. +Do not set `dataset-dir` or `model-dir` in normal SageMaker runs. SageMaker sets those automatically through `SM_CHANNEL_TRAIN` and `SM_MODEL_DIR`. diff --git a/examples/meter-detection/download_dataset.sh b/examples/meter-detection/download_dataset.sh deleted file mode 100755 index 77684eb..0000000 --- a/examples/meter-detection/download_dataset.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -WORKSPACE="kemals-workspace-kbc8l" -PROJECT="electric-meter-detection-o4tfi" -VERSION="1" -FORMAT="yolov8" -DATASET_DIR="examples/meter-detection/data/electric-meter-detection" - -if [[ -z "${ROBOFLOW_API_KEY:-}" ]]; then - echo "ROBOFLOW_API_KEY is required." >&2 - echo "Run: export ROBOFLOW_API_KEY=your-roboflow-api-key" >&2 - exit 1 -fi - -if ! command -v curl >/dev/null 2>&1; then - echo "curl is required." >&2 - exit 1 -fi - -if ! command -v unzip >/dev/null 2>&1; then - echo "unzip is required." >&2 - exit 1 -fi - -TMP_DIR="$(mktemp -d)" -trap 'rm -rf "${TMP_DIR}"' EXIT - -API_URL="https://api.roboflow.com/${WORKSPACE}/${PROJECT}/${VERSION}/${FORMAT}?api_key=${ROBOFLOW_API_KEY}" -RESPONSE_FILE="${TMP_DIR}/roboflow-export.json" -ZIP_FILE="${TMP_DIR}/dataset.zip" - -echo "Requesting Roboflow export link..." -curl -fsSL "${API_URL}" -o "${RESPONSE_FILE}" - -DOWNLOAD_URL="$( - sed -n 's/.*"link"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "${RESPONSE_FILE}" \ - | head -n 1 \ - | sed 's#\\/#/#g; s#\\u0026#\&#g' -)" - -if [[ -z "${DOWNLOAD_URL}" ]]; then - echo "Could not find export.link in Roboflow response." >&2 - echo "Response:" >&2 - cat "${RESPONSE_FILE}" >&2 - exit 1 -fi - -mkdir -p "${DATASET_DIR}" - -echo "Downloading dataset ZIP..." -curl -fL "${DOWNLOAD_URL}" -o "${ZIP_FILE}" - -echo "Extracting dataset..." -unzip -q -o "${ZIP_FILE}" -d "${DATASET_DIR}" - -echo "Downloaded dataset to ${DATASET_DIR}" diff --git a/examples/meter-detection/source/train.py b/examples/meter-detection/source/train.py index f27f537..b4454f7 100644 --- a/examples/meter-detection/source/train.py +++ b/examples/meter-detection/source/train.py @@ -24,37 +24,29 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--patience", type=int, default=20) parser.add_argument("--device", default=None) parser.add_argument("--data-yaml", default=None) - parser.add_argument("--train-dir", default=os.environ.get("SM_CHANNEL_TRAIN", "/opt/ml/input/data/train")) + parser.add_argument("--dataset-dir", default=os.environ.get("SM_CHANNEL_TRAIN", "/opt/ml/input/data/train")) + parser.add_argument("--train-dir", dest="dataset_dir", help=argparse.SUPPRESS) parser.add_argument("--model-dir", default=os.environ.get("SM_MODEL_DIR", "/opt/ml/model")) return parser.parse_args() -def find_data_yaml(train_dir: Path, explicit_path: str | None) -> Path: +def find_data_yaml(dataset_dir: Path, explicit_path: str | None) -> Path: if explicit_path: data_yaml = Path(explicit_path) if data_yaml.is_file(): return data_yaml raise FileNotFoundError(f"Configured data.yaml does not exist: {data_yaml}") - matches = sorted(train_dir.rglob("data.yaml")) + matches = sorted(dataset_dir.rglob("data.yaml")) if not matches: - raise FileNotFoundError(f"Could not find data.yaml under {train_dir}") + raise FileNotFoundError(f"Could not find data.yaml under {dataset_dir}") if len(matches) > 1: print(f"Found multiple data.yaml files; using {matches[0]}") return matches[0] -def _split_exists(dataset_root: Path, value: Any) -> bool: - if value is None: - return False - split_path = Path(str(value)) - if split_path.is_absolute(): - return split_path.exists() - return (dataset_root / split_path).exists() - - def prepare_data_yaml(data_yaml: Path) -> Path: - """Write a SageMaker-local data file with absolute dataset paths.""" + """Write a SageMaker-local data file rooted at the uploaded dataset.""" dataset_root = data_yaml.parent data = yaml.safe_load(data_yaml.read_text(encoding="utf-8")) if not isinstance(data, dict): @@ -62,24 +54,8 @@ def prepare_data_yaml(data_yaml: Path) -> Path: normalized = dict(data) normalized["path"] = str(dataset_root) - - for split_name in ("train", "val", "valid", "test"): - split_value = normalized.get(split_name) - if split_value is None: - continue - split_path = Path(str(split_value)) - if split_path.is_absolute(): - normalized[split_name] = str(split_path) - else: - normalized[split_name] = str((dataset_root / split_path).resolve()) - if "val" not in normalized and "valid" in normalized: - normalized["val"] = normalized["valid"] - - if not _split_exists(dataset_root, normalized.get("train")): - raise FileNotFoundError(f"Could not resolve train split from {data_yaml}") - if not _split_exists(dataset_root, normalized.get("val")): - raise FileNotFoundError(f"Could not resolve validation split from {data_yaml}") + normalized["val"] = normalized.pop("valid") prepared_path = dataset_root / "data.sagemaker.yaml" prepared_path.write_text(yaml.safe_dump(normalized, sort_keys=False), encoding="utf-8") @@ -95,11 +71,11 @@ def copy_if_exists(source: Path, destination: Path) -> None: def main() -> None: args = parse_args() - train_dir = Path(args.train_dir) + dataset_dir = Path(args.dataset_dir) model_dir = Path(args.model_dir) model_dir.mkdir(parents=True, exist_ok=True) - data_yaml = prepare_data_yaml(find_data_yaml(train_dir, args.data_yaml)) + data_yaml = prepare_data_yaml(find_data_yaml(dataset_dir, args.data_yaml)) model = YOLO(args.model) train_kwargs: dict[str, Any] = {