From 10519e1398407aff2e4db22e1f535479a7e2118c Mon Sep 17 00:00:00 2001
From: Kirk McKusick <mckusick@FreeBSD.org>
Date: Sat, 25 May 2019 00:07:49 +0000
Subject: [PATCH] When using the destroy option to shut down a nop GEOM module,
 I/O operations already in its queue were not being properly drained. The GEOM
 framework does the queue draining, but the module needs to wait for the
 draining to happen. The waiting is done by adding a g_nop_providergone()
 function to wait for the I/O operations to finish up. This change is similar
 to change -r345758 made to the memory-disk driver.

Submitted by: Chuck Silvers
Tested by:    Chuck Silvers
MFC after:    1 week
Sponsored by: Netflix
---
 sys/geom/nop/g_nop.c | 38 +++++++++++++++++++++++++-------------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/sys/geom/nop/g_nop.c b/sys/geom/nop/g_nop.c
index 889b4a2fe2dd..68d652e2cec6 100644
--- a/sys/geom/nop/g_nop.c
+++ b/sys/geom/nop/g_nop.c
@@ -54,17 +54,26 @@ static int g_nop_destroy_geom(struct gctl_req *req, struct g_class *mp,
     struct g_geom *gp);
 static void g_nop_config(struct gctl_req *req, struct g_class *mp,
     const char *verb);
-static void g_nop_dumpconf(struct sbuf *sb, const char *indent,
-    struct g_geom *gp, struct g_consumer *cp, struct g_provider *pp);
+static g_access_t g_nop_access;
+static g_dumpconf_t g_nop_dumpconf;
+static g_orphan_t g_nop_orphan;
+static g_provgone_t g_nop_providergone;
+static g_resize_t g_nop_resize;
+static g_start_t g_nop_start;
 
 struct g_class g_nop_class = {
 	.name = G_NOP_CLASS_NAME,
 	.version = G_VERSION,
 	.ctlreq = g_nop_config,
-	.destroy_geom = g_nop_destroy_geom
+	.destroy_geom = g_nop_destroy_geom,
+	.access = g_nop_access,
+	.dumpconf = g_nop_dumpconf,
+	.orphan = g_nop_orphan,
+	.providergone = g_nop_providergone,
+	.resize = g_nop_resize,
+	.start = g_nop_start,
 };
 
-
 static void
 g_nop_orphan(struct g_consumer *cp)
 {
@@ -320,11 +329,6 @@ g_nop_create(struct gctl_req *req, struct g_class *mp, struct g_provider *pp,
 	sc->sc_wrotebytes = 0;
 	mtx_init(&sc->sc_lock, "gnop lock", NULL, MTX_DEF);
 	gp->softc = sc;
-	gp->start = g_nop_start;
-	gp->orphan = g_nop_orphan;
-	gp->resize = g_nop_resize;
-	gp->access = g_nop_access;
-	gp->dumpconf = g_nop_dumpconf;
 
 	newpp = g_new_providerf(gp, "%s", gp->name);
 	newpp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE;
@@ -357,6 +361,18 @@ g_nop_create(struct gctl_req *req, struct g_class *mp, struct g_provider *pp,
 	return (error);
 }
 
+static void
+g_nop_providergone(struct g_provider *pp)
+{
+	struct g_geom *gp = pp->geom;
+	struct g_nop_softc *sc = gp->softc;
+
+	gp->softc = NULL;
+	free(sc->sc_physpath, M_GEOM);
+	mtx_destroy(&sc->sc_lock);
+	g_free(sc);
+}
+
 static int
 g_nop_destroy(struct g_geom *gp, boolean_t force)
 {
@@ -367,7 +383,6 @@ g_nop_destroy(struct g_geom *gp, boolean_t force)
 	sc = gp->softc;
 	if (sc == NULL)
 		return (ENXIO);
-	free(sc->sc_physpath, M_GEOM);
 	pp = LIST_FIRST(&gp->provider);
 	if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
 		if (force) {
@@ -381,9 +396,6 @@ g_nop_destroy(struct g_geom *gp, boolean_t force)
 	} else {
 		G_NOP_DEBUG(0, "Device %s removed.", gp->name);
 	}
-	gp->softc = NULL;
-	mtx_destroy(&sc->sc_lock);
-	g_free(sc);
 	g_wither_geom(gp, ENXIO);
 
 	return (0);