Index: lib/geom/concat/geom_concat.c =================================================================== --- lib/geom/concat/geom_concat.c (revision 344217) +++ lib/geom/concat/geom_concat.c (working copy) @@ -59,6 +59,9 @@ { "create", G_FLAG_VERBOSE | G_FLAG_LOADKLD, NULL, G_NULL_OPTS, "[-v] name prov ..." }, + { "append", G_FLAG_VERBOSE, NULL, G_NULL_OPTS, + "[-v] name prov" + }, { "destroy", G_FLAG_VERBOSE, NULL, { { 'f', "force", NULL, G_TYPE_BOOL }, Index: sys/geom/concat/g_concat.c =================================================================== --- sys/geom/concat/g_concat.c (revision 344217) +++ sys/geom/concat/g_concat.c (working copy) @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -104,6 +105,8 @@ { u_int i, no; + sx_assert(&sc->sc_lock_append, SA_LOCKED); + no = 0; for (i = 0; i < sc->sc_ndisks; i++) { if (sc->sc_disks[i].d_consumer != NULL) @@ -172,10 +175,12 @@ struct g_consumer *cp1, *cp2, *tmp; struct g_concat_disk *disk; struct g_geom *gp; + struct g_concat_softc *sc; int error; g_topology_assert(); gp = pp->geom; + sc = gp->softc; /* On first open, grab an extra "exclusive" bit */ if (pp->acr == 0 && pp->acw == 0 && pp->ace == 0) @@ -184,6 +189,7 @@ if ((pp->acr + dr) == 0 && (pp->acw + dw) == 0 && (pp->ace + de) == 0) de--; + sx_slock(&sc->sc_lock_append); LIST_FOREACH_SAFE(cp1, &gp->consumer, consumer, tmp) { error = g_access(cp1, dr, dw, de); if (error != 0) @@ -194,9 +200,11 @@ g_concat_remove_disk(disk); /* May destroy geom. */ } } + sx_sunlock(&sc->sc_lock_append); return (0); fail: + sx_sunlock(&sc->sc_lock_append); LIST_FOREACH(cp2, &gp->consumer, consumer) { if (cp1 == cp2) break; @@ -213,6 +221,7 @@ int i, val; sc = bp->bio_to->geom->softc; + sx_assert(&sc->sc_lock_append, SX_LOCKED); for (i = 0; i < sc->sc_ndisks; i++) { disk = &sc->sc_disks[i]; if (!disk->d_removed && disk->d_candelete) @@ -284,6 +293,8 @@ struct bio *cbp; u_int no; + sx_assert(&sc->sc_lock_append, SX_LOCKED); + bioq_init(&queue); for (no = 0; no < sc->sc_ndisks; no++) { cbp = g_clone_bio(bp); @@ -331,6 +342,7 @@ bp->bio_to->error, bp->bio_to->name)); G_CONCAT_LOGREQ(bp, "Request received."); + sx_slock(&sc->sc_lock_append); switch (bp->bio_cmd) { case BIO_READ: @@ -339,20 +351,20 @@ break; case BIO_FLUSH: g_concat_flush(sc, bp); - return; + goto end; case BIO_GETATTR: if (strcmp("GEOM::kerneldump", bp->bio_attribute) == 0) { g_concat_kernel_dump(bp); - return; + goto end; } else if (strcmp("GEOM::candelete", bp->bio_attribute) == 0) { g_concat_candelete(bp); - return; + goto end; } /* To which provider it should be delivered? */ /* FALLTHROUGH */ default: g_io_deliver(bp, EOPNOTSUPP); - return; + goto end; } offset = bp->bio_offset; @@ -383,7 +395,7 @@ if (bp->bio_error == 0) bp->bio_error = ENOMEM; g_io_deliver(bp, bp->bio_error); - return; + goto end; } bioq_insert_tail(&queue, cbp); /* @@ -419,6 +431,8 @@ cbp->bio_caller1 = NULL; g_io_request(cbp, disk->d_consumer); } +end: + sx_sunlock(&sc->sc_lock_append); } static void @@ -519,14 +533,23 @@ int error; g_topology_assert(); + + sx_slock(&sc->sc_lock_append); + /* Metadata corrupted? */ if (no >= sc->sc_ndisks) + { + sx_sunlock(&sc->sc_lock_append); return (EINVAL); + } disk = &sc->sc_disks[no]; /* Check if disk is not already attached. */ if (disk->d_consumer != NULL) + { + sx_sunlock(&sc->sc_lock_append); return (EEXIST); + } gp = sc->sc_geom; fcp = LIST_FIRST(&gp->consumer); @@ -535,6 +558,7 @@ cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE; error = g_attach(cp, pp); if (error != 0) { + sx_sunlock(&sc->sc_lock_append); g_destroy_consumer(cp); return (error); } @@ -542,6 +566,7 @@ if (fcp != NULL && (fcp->acr > 0 || fcp->acw > 0 || fcp->ace > 0)) { error = g_access(cp, fcp->acr, fcp->acw, fcp->ace); if (error != 0) { + sx_sunlock(&sc->sc_lock_append); g_detach(cp); g_destroy_consumer(cp); return (error); @@ -550,8 +575,15 @@ if (sc->sc_type == G_CONCAT_TYPE_AUTOMATIC) { struct g_concat_metadata md; + // temporarilygive up the lock to avoid lock order violation + // due to topology unlock in g_concat_read_metadata + sx_sunlock(&sc->sc_lock_append); /* Re-read metadata. */ error = g_concat_read_metadata(cp, &md); + sx_slock(&sc->sc_lock_append); + // re-compute disk ptr in case sc_disks was reallocated + disk = &sc->sc_disks[no]; + if (error != 0) goto fail; @@ -573,9 +605,11 @@ G_CONCAT_DEBUG(0, "Disk %s attached to %s.", pp->name, sc->sc_name); g_concat_check_and_run(sc); + sx_sunlock(&sc->sc_lock_append); // need lock for check_and_run return (0); fail: + sx_sunlock(&sc->sc_lock_append); if (fcp != NULL && (fcp->acr > 0 || fcp->acw > 0 || fcp->ace > 0)) g_access(cp, -fcp->acr, -fcp->acw, -fcp->ace); g_detach(cp); @@ -623,6 +657,7 @@ sc->sc_disks[no].d_consumer = NULL; sc->sc_type = type; mtx_init(&sc->sc_lock, "gconcat lock", NULL, MTX_DEF); + sx_init(&sc->sc_lock_append, "gconcat append lock"); gp->softc = sc; sc->sc_geom = gp; @@ -672,6 +707,7 @@ gp->name)); free(sc->sc_disks, M_CONCAT); mtx_destroy(&sc->sc_lock); + sx_destroy(&sc->sc_lock_append); free(sc, M_CONCAT); G_CONCAT_DEBUG(0, "Device %s destroyed.", gp->name); @@ -947,7 +983,199 @@ } } +static struct g_concat_disk * +g_concat_find_disk(struct g_concat_softc *sc, const char *name) +{ + u_int i; + struct g_concat_disk *disk; + + sx_assert(&sc->sc_lock_append, SX_LOCKED); + if (strncmp(name, "/dev/", 5) == 0) + name += 5; + for (i = 0; i < sc->sc_ndisks; i++) { + disk = &sc->sc_disks[i]; + if (disk->d_consumer == NULL) + continue; + if (disk->d_consumer->provider == NULL) + continue; + if (strcmp(disk->d_consumer->provider->name, name) == 0) + return (disk); + } + return (NULL); +} + static void +g_concat_ctl_append(struct gctl_req *req, struct g_class *mp) +{ + u_int no, i; + struct g_concat_softc *sc; + struct g_provider *pp; + struct g_consumer *cp, *fcp; // no idea what fcp does + struct g_geom *gp; + const char *name, *cname; + //char param[16]; + struct g_concat_disk *disks_tmp, *disk; + int *nargs; + int error; + int disk_candelete; + struct g_concat_metadata md; + u_char *sector; + + g_topology_assert(); + + nargs = gctl_get_paraml(req, "nargs", sizeof(*nargs)); + if (nargs == NULL) { + gctl_error(req, "No '%s' argument.", "nargs"); + return; + } + if (*nargs != 2) { + gctl_error(req, "Invalid number of arguments."); + return; + } + + cname = gctl_get_asciiparam(req, "arg0"); + if (cname == NULL) { + gctl_error(req, "No 'arg%u' argument.", 0); + return; + } + sc = g_concat_find_device(mp, cname); + if (sc == NULL) { + gctl_error(req, "No such device: %s.", cname); + return; + } + if (sc->sc_provider == NULL) { + // this won't race with g_concat_remove_disk as both + // are holding the topology lock + gctl_error(req, "Device not active, can't append: %s.", cname); + return; + } + G_CONCAT_DEBUG(1, "Appending to %s:", cname); + sx_xlock(&sc->sc_lock_append); + gp = sc->sc_geom; + fcp = LIST_FIRST(&gp->consumer); + + name = gctl_get_asciiparam(req, "arg1"); + if (name == NULL) { + gctl_error(req, "No 'arg%u' argument.", 1); + goto fail; + } + if (strncmp(name, "/dev/", strlen("/dev/")) == 0) + name += strlen("/dev/"); + pp = g_provider_by_name(name); + if (pp == NULL) { + G_CONCAT_DEBUG(1, "Disk %s is invalid.", name); + gctl_error(req, "Disk %s is invalid.", name); + goto fail; + } + G_CONCAT_DEBUG(1, "Appending %s to this", name); + + if (g_concat_find_disk(sc, name) != NULL) { + gctl_error(req, "Disk %s already appended.", name); + goto fail; + } + + // make sure sector size matches + if ((sc->sc_provider->sectorsize % pp->sectorsize) != 0) { + gctl_error(req, "Providers sectorsize mismatch: %u vs %u", + sc->sc_provider->sectorsize, pp->sectorsize); + goto fail; + } + + cp = g_new_consumer(gp); + cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE; + error = g_attach(cp, pp); + if (error != 0) { + g_destroy_consumer(cp); + gctl_error(req, "Cannot open device %s (error=%d).", + name, error); + goto fail; + } + + error = g_access(cp, 1, 0, 0); + if (error == 0) { + error = g_getattr("GEOM::candelete", cp, &disk_candelete); + if (error != 0) + disk_candelete = 0; + (void)g_access(cp, -1, 0, 0); + } else + G_CONCAT_DEBUG(1, "Failed to access disk %s, error %d.", name, error); + + // invoke g_access exactly as deep as all the other members currently are + if (fcp != NULL && (fcp->acr > 0 || fcp->acw > 0 || fcp->ace > 0)) { + error = g_access(cp, fcp->acr, fcp->acw, fcp->ace); + if (error != 0) { + g_detach(cp); + g_destroy_consumer(cp); + gctl_error(req, "Failed to access disk %s (error=%d).", name, error); + goto fail; + } + } + + disks_tmp = sc->sc_disks; + sc->sc_disks = malloc(sizeof(struct g_concat_disk) * (sc->sc_ndisks + 1), M_CONCAT, M_WAITOK | M_ZERO); + memcpy(sc->sc_disks, disks_tmp, sizeof(struct g_concat_disk) * sc->sc_ndisks); + // fix up disk ptrs of existing consumers + for (i = 0; i < sc->sc_ndisks; i++) { + sc->sc_disks[i].d_consumer->private = &sc->sc_disks[i]; + } + // now it is safe to free the old array + free(disks_tmp, M_CONCAT); + disks_tmp = NULL; + + disk = &sc->sc_disks[sc->sc_ndisks]; + cp->private = disk; + disk->d_consumer = cp; + disk->d_softc = sc; + disk->d_start = sc->sc_disks[sc->sc_ndisks - 1].d_end; + disk->d_end = disk->d_start + cp->provider->mediasize; + disk->d_candelete = disk_candelete; + disk->d_removed = 0; + + sc->sc_ndisks += 1; + + if (sc->sc_type == G_CONCAT_TYPE_AUTOMATIC) { + // last sector is for metadata + disk->d_end -= cp->provider->sectorsize; + + // update metadata on all parts + strlcpy(md.md_magic, G_CONCAT_MAGIC, sizeof(md.md_magic)); + md.md_version = G_CONCAT_VERSION; + strlcpy(md.md_name, cname, sizeof(md.md_name)); + md.md_id = sc->sc_id; + md.md_all = sc->sc_ndisks; + for (no = 0; no < sc->sc_ndisks; no++) { + disk = &sc->sc_disks[no]; + + md.md_no = no; + // TODO: implement hardcode? (not sure how, not sure why it even exists) + bzero(md.md_provider, sizeof(md.md_provider)); + md.md_provsize = disk->d_consumer->provider->mediasize; + + pp = disk->d_consumer->provider; + sector = g_malloc(pp->sectorsize, M_WAITOK); + + concat_metadata_encode(&md, sector); + error = g_access(disk->d_consumer, 0, 1, 0); + if (error == 0) { + error = g_write_data(disk->d_consumer, pp->mediasize - pp->sectorsize, + sector, pp->sectorsize); + } + (void)g_access(disk->d_consumer, 0, -1, 0); + g_free(sector); + if (error != 0) { + gctl_error(req, "Cannot store metadata on %s.", pp->name); + } + } + + } + + g_resize_provider(sc->sc_provider, disk->d_end); + +fail: + sx_xunlock(&sc->sc_lock_append); +} + +static void g_concat_config(struct gctl_req *req, struct g_class *mp, const char *verb) { uint32_t *version; @@ -971,6 +1199,9 @@ strcmp(verb, "stop") == 0) { g_concat_ctl_destroy(req, mp); return; + } else if (strcmp(verb, "append") == 0) { + g_concat_ctl_append(req, mp); + return; } gctl_error(req, "Unknown verb."); } @@ -985,6 +1216,8 @@ sc = gp->softc; if (sc == NULL) return; + + sx_slock(&sc->sc_lock_append); if (pp != NULL) { /* Nothing here. */ } else if (cp != NULL) { @@ -992,7 +1225,7 @@ disk = cp->private; if (disk == NULL) - return; + goto end; sbuf_printf(sb, "%s%jd\n", indent, (intmax_t)disk->d_end); sbuf_printf(sb, "%s%jd\n", indent, @@ -1021,6 +1254,8 @@ sbuf_printf(sb, "DOWN"); sbuf_printf(sb, "\n"); } +end: + sx_sunlock(&sc->sc_lock_append); } DECLARE_GEOM_CLASS(g_concat_class, g_concat); Index: sys/geom/concat/g_concat.h =================================================================== --- sys/geom/concat/g_concat.h (revision 344217) +++ sys/geom/concat/g_concat.h (working copy) @@ -42,8 +42,9 @@ * 2 - Added 'stop' command to gconcat(8). * 3 - Added md_provider field to metadata and '-h' option to gconcat(8). * 4 - Added md_provsize field to metadata. + * 5 - Added online append functionality ('append' command). */ -#define G_CONCAT_VERSION 4 +#define G_CONCAT_VERSION 5 #ifdef _KERNEL #define G_CONCAT_TYPE_MANUAL 0 @@ -87,6 +88,7 @@ struct g_concat_disk *sc_disks; uint16_t sc_ndisks; struct mtx sc_lock; + struct sx sc_lock_append; /* controls reallocation of sc_disks */ }; #define sc_name sc_geom->name #endif /* _KERNEL */